mirror of
https://github.com/supermemoryai/supermemory.git
synced 2026-05-12 14:10:55 +00:00
Import twitter bookmarks
This commit is contained in:
Dhravya
parent
f8068a6e01
commit
fc7ec15263
5 changed files with 438 additions and 8 deletions
9
apps/cf-ai-backend/src/env.d.ts
vendored
9
apps/cf-ai-backend/src/env.d.ts
vendored
|
|
@ -4,4 +4,13 @@ interface Env {
|
|||
SECURITY_KEY: string;
|
||||
OPENAI_API_KEY: string;
|
||||
GOOGLE_AI_API_KEY: string;
|
||||
MY_QUEUE: Queue<TweetData>;
|
||||
}
|
||||
|
||||
interface TweetData {
|
||||
tweetText: string;
|
||||
postUrl: string;
|
||||
authorName: string;
|
||||
handle: string;
|
||||
time: string;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,10 @@ index_name = "any-vector"
|
|||
[ai]
|
||||
binding = "AI"
|
||||
|
||||
[[queues.producers]]
|
||||
queue = "batch-vector-queue"
|
||||
binding = "MY_QUEUE"
|
||||
|
||||
# Variable bindings. These are arbitrary, plaintext strings (similar to environment variables)
|
||||
# Note: Use secrets to store sensitive data.
|
||||
# Docs: https://developers.cloudflare.com/workers/platform/environment-variables
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ function App() {
|
|||
null,
|
||||
);
|
||||
|
||||
const doStuff = () => {
|
||||
const getUserData = () => {
|
||||
chrome.runtime.sendMessage({ type: "getJwt" }, (response) => {
|
||||
const jwt = response.jwt;
|
||||
const loginButton = document.getElementById("login");
|
||||
|
|
@ -41,9 +41,69 @@ function App() {
|
|||
};
|
||||
|
||||
useEffect(() => {
|
||||
doStuff();
|
||||
getUserData();
|
||||
}, []);
|
||||
|
||||
// TODO: Implement getting bookmarks from API directly
|
||||
// const [status, setStatus] = useState('');
|
||||
// const [bookmarks, setBookmarks] = useState<TweetData[]>([]);
|
||||
|
||||
// const fetchBookmarks = (e: React.MouseEvent<HTMLButtonElement>) => {
|
||||
// e.preventDefault();
|
||||
|
||||
// chrome.tabs.query({ active: true, currentWindow: true }, function (tabs) {
|
||||
// chrome.tabs.sendMessage(tabs[0].id!, { action: 'showProgressIndicator' });
|
||||
// });
|
||||
|
||||
// chrome.tabs.create(
|
||||
// { url: 'https://twitter.com/i/bookmarks/all' },
|
||||
// function (tab) {
|
||||
// chrome.tabs.onUpdated.addListener(function listener(tabId, info) {
|
||||
// if (tabId === tab.id && info.status === 'complete') {
|
||||
// chrome.tabs.onUpdated.removeListener(listener);
|
||||
|
||||
// chrome.runtime.sendMessage(
|
||||
// { action: 'getAuthData' },
|
||||
// function (response) {
|
||||
// const authorizationHeader = response.authorizationHeader;
|
||||
// const csrfToken = response.csrfToken;
|
||||
// const cookies = response.cookies;
|
||||
|
||||
// if (authorizationHeader && csrfToken && cookies) {
|
||||
// fetchAllBookmarks(authorizationHeader, csrfToken, cookies)
|
||||
// .then((bookmarks) => {
|
||||
// console.log('Bookmarks data:', bookmarks);
|
||||
// setBookmarks(bookmarks);
|
||||
// chrome.tabs.sendMessage(tabId, {
|
||||
// action: 'hideProgressIndicator',
|
||||
// });
|
||||
// setStatus(
|
||||
// `Fetched ${bookmarks.length} bookmarked tweets.`,
|
||||
// );
|
||||
// })
|
||||
// .catch((error) => {
|
||||
// console.error('Error:', error);
|
||||
// chrome.tabs.sendMessage(tabId, {
|
||||
// action: 'hideProgressIndicator',
|
||||
// });
|
||||
// setStatus(
|
||||
// 'Error fetching bookmarks. Please check the console for details.',
|
||||
// );
|
||||
// });
|
||||
// } else {
|
||||
// chrome.tabs.sendMessage(tabId, {
|
||||
// action: 'hideProgressIndicator',
|
||||
// });
|
||||
// setStatus('Missing authentication data');
|
||||
// }
|
||||
// },
|
||||
// );
|
||||
// }
|
||||
// });
|
||||
// },
|
||||
// );
|
||||
// };
|
||||
|
||||
return (
|
||||
<div className="p-8">
|
||||
<button
|
||||
|
|
@ -69,6 +129,19 @@ function App() {
|
|||
<h3>{userData.data.user.name}</h3>
|
||||
<p>{userData.data.user.email}</p>
|
||||
</div>
|
||||
{/* TODO: Implement getting bookmarks from API directly */}
|
||||
{/* <button onClick={(e) => fetchBookmarks(e)}>Fetch Bookmarks</button>
|
||||
<div>{status}</div>
|
||||
|
||||
<div>
|
||||
{bookmarks.map((bookmark) => (
|
||||
<div key={bookmark.tweet_id}>
|
||||
<p>{bookmark.author}</p>
|
||||
<p>{bookmark.date}</p>
|
||||
<p>{bookmark.full_text}</p>
|
||||
</div>
|
||||
))}
|
||||
</div> */}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
|
@ -76,4 +149,160 @@ function App() {
|
|||
);
|
||||
}
|
||||
|
||||
// TODO: Implement getting bookmarks from API directly
|
||||
// async function fetchAllBookmarks(
|
||||
// authorizationHeader: string,
|
||||
// csrfToken: string,
|
||||
// cookies: string,
|
||||
// ): Promise<TweetData[]> {
|
||||
// const baseUrl =
|
||||
// 'https://twitter.com/i/api/graphql/uJEL6XARgGmo2EAsO2Pfkg/Bookmarks';
|
||||
// const params = new URLSearchParams({
|
||||
// variables: JSON.stringify({
|
||||
// count: 100,
|
||||
// includePromotedContent: true,
|
||||
// }),
|
||||
// features: JSON.stringify({
|
||||
// graphql_timeline_v2_bookmark_timeline: true,
|
||||
// rweb_tipjar_consumption_enabled: false,
|
||||
// responsive_web_graphql_exclude_directive_enabled: true,
|
||||
// verified_phone_label_enabled: true,
|
||||
// creator_subscriptions_tweet_preview_api_enabled: true,
|
||||
// responsive_web_graphql_timeline_navigation_enabled: true,
|
||||
// responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
||||
// communities_web_enable_tweet_community_results_fetch: true,
|
||||
// c9s_tweet_anatomy_moderator_badge_enabled: true,
|
||||
// tweetypie_unmention_optimization_enabled: true,
|
||||
// responsive_web_edit_tweet_api_enabled: true,
|
||||
// graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
||||
// view_counts_everywhere_api_enabled: true,
|
||||
// longform_notetweets_consumption_enabled: true,
|
||||
// responsive_web_twitter_article_tweet_consumption_enabled: true,
|
||||
// tweet_awards_web_tipping_enabled: false,
|
||||
// creator_subscriptions_quote_tweet_preview_enabled: false,
|
||||
// freedom_of_speech_not_reach_fetch_enabled: true,
|
||||
// standardized_nudges_misinfo: true,
|
||||
// tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled:
|
||||
// true,
|
||||
// tweet_with_visibility_results_prefer_gql_media_interstitial_enabled:
|
||||
// false,
|
||||
// rweb_video_timestamps_enabled: true,
|
||||
// longform_notetweets_rich_text_read_enabled: true,
|
||||
// longform_notetweets_inline_media_enabled: true,
|
||||
// responsive_web_enhance_cards_enabled: false,
|
||||
// }),
|
||||
// });
|
||||
|
||||
// const requestUrl = `${baseUrl}?${params}`;
|
||||
|
||||
// const headers = {
|
||||
// Authorization: authorizationHeader,
|
||||
// 'X-Csrf-Token': csrfToken,
|
||||
// Cookie: cookies,
|
||||
// };
|
||||
|
||||
// const bookmarks: TweetData[] = [];
|
||||
// let nextCursor = null;
|
||||
// let requestCount = 0;
|
||||
// const maxRequestsPerWindow = 450;
|
||||
// const windowDuration = 15 * 60 * 1000; // 15 minutes in milliseconds
|
||||
// let windowStartTime = Date.now();
|
||||
|
||||
// do {
|
||||
// if (nextCursor) {
|
||||
// params.set(
|
||||
// 'variables',
|
||||
// JSON.stringify({
|
||||
// count: 100,
|
||||
// cursor: nextCursor,
|
||||
// includePromotedContent: true,
|
||||
// }),
|
||||
// );
|
||||
// }
|
||||
|
||||
// // Check if the rate limit is exceeded
|
||||
// if (requestCount >= maxRequestsPerWindow) {
|
||||
// const elapsedTime = Date.now() - windowStartTime;
|
||||
// if (elapsedTime < windowDuration) {
|
||||
// const waitTime = windowDuration - elapsedTime;
|
||||
// await new Promise((resolve) => setTimeout(resolve, waitTime));
|
||||
// }
|
||||
// requestCount = 0;
|
||||
// windowStartTime = Date.now();
|
||||
// }
|
||||
|
||||
// try {
|
||||
// const response = await fetch(requestUrl, {
|
||||
// method: 'GET',
|
||||
// headers: headers,
|
||||
// });
|
||||
|
||||
// requestCount++;
|
||||
|
||||
// if (!response.ok) {
|
||||
// throw new Error(`HTTP error! status: ${response.status}`);
|
||||
// }
|
||||
|
||||
// const data = await response.json();
|
||||
// const timeline = data.data.bookmark_timeline_v2.timeline;
|
||||
|
||||
// timeline.instructions.forEach(
|
||||
// (instruction: {
|
||||
// type: string;
|
||||
// entries: {
|
||||
// content: {
|
||||
// entryType: string;
|
||||
// itemContent: {
|
||||
// tweet_results: {
|
||||
// result: {
|
||||
// legacy: {
|
||||
// full_text: string;
|
||||
// created_at: string;
|
||||
// };
|
||||
// core: {
|
||||
// user_results: {
|
||||
// result: {
|
||||
// legacy: {
|
||||
// screen_name: string;
|
||||
// };
|
||||
// };
|
||||
// };
|
||||
// };
|
||||
// rest_id: string;
|
||||
// };
|
||||
// };
|
||||
// };
|
||||
// };
|
||||
// }[];
|
||||
// }) => {
|
||||
// if (instruction.type === 'TimelineAddEntries') {
|
||||
// instruction.entries.forEach((entry) => {
|
||||
// if (entry.content.entryType === 'TimelineTimelineItem') {
|
||||
// const tweet = entry.content.itemContent.tweet_results.result;
|
||||
// const tweetData = {
|
||||
// full_text: tweet.legacy.full_text,
|
||||
// url: `https://twitter.com/${tweet.core.user_results.result.legacy.screen_name}/status/${tweet.rest_id}`,
|
||||
// author: tweet.core.user_results.result.legacy.screen_name,
|
||||
// date: tweet.legacy.created_at,
|
||||
// tweet_id: tweet.rest_id,
|
||||
// };
|
||||
// bookmarks.push(tweetData);
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
// },
|
||||
// );
|
||||
|
||||
// nextCursor = timeline.instructions.find(
|
||||
// (instruction: { type: string }) =>
|
||||
// instruction.type === 'TimelineTerminateTimeline',
|
||||
// )?.direction?.cursor;
|
||||
// } catch (error) {
|
||||
// console.error('Error fetching bookmarks:', error);
|
||||
// throw error;
|
||||
// }
|
||||
// } while (nextCursor);
|
||||
|
||||
// return bookmarks;
|
||||
// }
|
||||
export default App;
|
||||
|
|
|
|||
|
|
@ -25,20 +25,176 @@ function sendUrlToAPI() {
|
|||
}
|
||||
|
||||
function SideBar() {
|
||||
// TODO: Implement getting bookmarks from API directly
|
||||
// chrome.runtime.onMessage.addListener(function (request) {
|
||||
// if (request.action === 'showProgressIndicator') {
|
||||
// // TODO: SHOW PROGRESS INDICATOR
|
||||
// // showProgressIndicator();
|
||||
// } else if (request.action === 'hideProgressIndicator') {
|
||||
// // hideProgressIndicator();
|
||||
// }
|
||||
// });
|
||||
|
||||
const [savedWebsites, setSavedWebsites] = useState<string[]>([]);
|
||||
|
||||
const [isSendingData, setIsSendingData] = useState(false);
|
||||
|
||||
interface TweetData {
|
||||
tweetText: string;
|
||||
postUrl: string;
|
||||
authorName: string;
|
||||
handle: string;
|
||||
time: string;
|
||||
}
|
||||
|
||||
const fetchBookmarks = () => {
|
||||
const tweets: TweetData[] = []; // Initialize an empty array to hold all tweet elements
|
||||
|
||||
const scrollInterval = 1000;
|
||||
const scrollStep = 5000; // Pixels to scroll on each step
|
||||
|
||||
let previousTweetCount = 0;
|
||||
let unchangedCount = 0;
|
||||
|
||||
const scrollToEndIntervalID = setInterval(() => {
|
||||
window.scrollBy(0, scrollStep);
|
||||
const currentTweetCount = tweets.length;
|
||||
if (currentTweetCount === previousTweetCount) {
|
||||
unchangedCount++;
|
||||
if (unchangedCount >= 2) {
|
||||
// Stop if the count has not changed 5 times
|
||||
console.log("Scraping complete");
|
||||
console.log("Total tweets scraped: ", tweets.length);
|
||||
console.log("Downloading tweets as JSON...");
|
||||
clearInterval(scrollToEndIntervalID); // Stop scrolling
|
||||
observer.disconnect(); // Stop observing DOM changes
|
||||
downloadTweetsAsJson(tweets); // Download the tweets list as a JSON file
|
||||
}
|
||||
} else {
|
||||
unchangedCount = 0; // Reset counter if new tweets were added
|
||||
}
|
||||
previousTweetCount = currentTweetCount; // Update previous count for the next check
|
||||
}, scrollInterval);
|
||||
|
||||
function updateTweets() {
|
||||
document
|
||||
.querySelectorAll('article[data-testid="tweet"]')
|
||||
.forEach((tweetElement) => {
|
||||
const authorName = (
|
||||
tweetElement.querySelector(
|
||||
'[data-testid="User-Name"]',
|
||||
) as HTMLElement
|
||||
)?.innerText;
|
||||
|
||||
const handle = (
|
||||
tweetElement.querySelector('[role="link"]') as HTMLLinkElement
|
||||
).href
|
||||
.split("/")
|
||||
.pop();
|
||||
|
||||
const tweetText = (
|
||||
tweetElement.querySelector(
|
||||
'[data-testid="tweetText"]',
|
||||
) as HTMLElement
|
||||
)?.innerText;
|
||||
const time = (
|
||||
tweetElement.querySelector("time") as HTMLTimeElement
|
||||
).getAttribute("datetime");
|
||||
const postUrl = (
|
||||
tweetElement.querySelector(
|
||||
".css-175oi2r.r-18u37iz.r-1q142lx a",
|
||||
) as HTMLLinkElement
|
||||
)?.href;
|
||||
|
||||
const isTweetNew = !tweets.some((tweet) => tweet.postUrl === postUrl);
|
||||
if (isTweetNew) {
|
||||
tweets.push({
|
||||
authorName,
|
||||
handle: handle ?? "",
|
||||
tweetText,
|
||||
time: time ?? "",
|
||||
postUrl,
|
||||
});
|
||||
console.log("Tweets capturados: ", tweets.length);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Initially populate the tweets array
|
||||
updateTweets();
|
||||
|
||||
// Create a MutationObserver to observe changes in the DOM
|
||||
const observer = new MutationObserver((mutations) => {
|
||||
mutations.forEach((mutation) => {
|
||||
if (mutation.addedNodes.length) {
|
||||
updateTweets(); // Call updateTweets whenever new nodes are added to the DOM
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Start observing the document body for child list changes
|
||||
observer.observe(document.body, { childList: true, subtree: true });
|
||||
|
||||
function downloadTweetsAsJson(tweetsArray: TweetData[]) {
|
||||
const jsonData = JSON.stringify(tweetsArray); // Convert the array to JSON
|
||||
|
||||
// TODO: SEND jsonData to server
|
||||
console.log(jsonData);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<TooltipProvider>
|
||||
<div className="anycontext-flex anycontext-flex-col anycontext-gap-2 anycontext-fixed anycontext-bottom-12 anycontext-right-0 anycontext-z-[99999] anycontext-font-sans">
|
||||
{/* <Tooltip delayDuration={300}>
|
||||
<TooltipContent side="left">
|
||||
<p>Open Sidebar</p>
|
||||
</TooltipContent>
|
||||
</Tooltip> */}
|
||||
<div className="anycontext-flex anycontext-group anycontext-flex-col anycontext-gap-2 anycontext-fixed anycontext-bottom-12 anycontext-right-0 anycontext-z-[99999] anycontext-font-sans">
|
||||
{window.location.href.includes("twitter.com") ||
|
||||
window.location.href.includes("x.com") ? (
|
||||
<Tooltip delayDuration={300}>
|
||||
<TooltipTrigger
|
||||
className="anycontext-bg-transparent
|
||||
anycontext-border-none anycontext-m-0 anycontext-p-0"
|
||||
>
|
||||
<button
|
||||
onClick={() => {
|
||||
if (window.location.href.endsWith("/i/bookmarks/all")) {
|
||||
fetchBookmarks();
|
||||
} else {
|
||||
window.location.href =
|
||||
"https://twitter.com/i/bookmarks/all";
|
||||
|
||||
setTimeout(() => {
|
||||
fetchBookmarks();
|
||||
}, 2500);
|
||||
}
|
||||
}}
|
||||
className="anycontext-open-button disabled:anycontext-opacity-30 anycontext-bg-transparent
|
||||
anycontext-border-none anycontext-m-0 anycontext-p-0"
|
||||
>
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
strokeWidth={1.5}
|
||||
stroke="currentColor"
|
||||
className="anycontext-w-6 anycontext-h-6"
|
||||
>
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M17.593 3.322c1.1.128 1.907 1.077 1.907 2.185V21L12 17.25 4.5 21V5.507c0-1.108.806-2.057 1.907-2.185a48.507 48.507 0 0 1 11.186 0Z"
|
||||
/>
|
||||
</svg>
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent className="anycontext-p-0" side="left">
|
||||
<p className="anycontext-p-0 anycontext-m-0">
|
||||
Import twitter bookmarks
|
||||
</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
) : (
|
||||
<></>
|
||||
)}
|
||||
<Tooltip delayDuration={300}>
|
||||
<TooltipTrigger
|
||||
className="anycontext-bg-transparent
|
||||
|
|
|
|||
|
|
@ -5,6 +5,30 @@ const backendUrl =
|
|||
? "http://localhost:3000"
|
||||
: "https://supermemory.dhr.wtf";
|
||||
|
||||
// TODO: Implement getting bookmarks from API directly
|
||||
// let authorizationHeader: string | null = null;
|
||||
// let csrfToken: string | null = null;
|
||||
// let cookies: string | null = null;
|
||||
|
||||
// chrome.webRequest.onBeforeSendHeaders.addListener(
|
||||
// (details) => {
|
||||
// for (let i = 0; i < details.requestHeaders!.length; ++i) {
|
||||
// const header = details.requestHeaders![i];
|
||||
// if (header.name.toLowerCase() === 'authorization') {
|
||||
// authorizationHeader = header.value || null;
|
||||
// } else if (header.name.toLowerCase() === 'x-csrf-token') {
|
||||
// csrfToken = header.value || null;
|
||||
// } else if (header.name.toLowerCase() === 'cookie') {
|
||||
// cookies = header.value || null;
|
||||
// }
|
||||
|
||||
// console.log(header, authorizationHeader, csrfToken, cookies)
|
||||
// }
|
||||
// },
|
||||
// { urls: ['https://twitter.com/*', 'https://x.com/*'] },
|
||||
// ['requestHeaders']
|
||||
// );
|
||||
|
||||
chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
|
||||
if (request.type === "getJwt") {
|
||||
chrome.storage.local.get(["jwt"], ({ jwt }) => {
|
||||
|
|
@ -73,4 +97,12 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
|
|||
return true;
|
||||
})();
|
||||
}
|
||||
// TODO: Implement getting bookmarks from API directly
|
||||
// else if (request.action === 'getAuthData') {
|
||||
// sendResponse({
|
||||
// authorizationHeader: authorizationHeader,
|
||||
// csrfToken: csrfToken,
|
||||
// cookies: cookies
|
||||
// });
|
||||
// }
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue