From d11b6361138b67eac8b556a0283366a30cc86009 Mon Sep 17 00:00:00 2001 From: Muhamad Aji Wibisono Date: Mon, 2 Jun 2025 20:26:14 +0700 Subject: [PATCH] feat: removed limit on message fetching --- surfsense_backend/app/connectors/discord_connector.py | 6 ++---- surfsense_backend/app/tasks/connectors_indexing_tasks.py | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/surfsense_backend/app/connectors/discord_connector.py b/surfsense_backend/app/connectors/discord_connector.py index 6d66b05..b737e63 100644 --- a/surfsense_backend/app/connectors/discord_connector.py +++ b/surfsense_backend/app/connectors/discord_connector.py @@ -142,7 +142,6 @@ class DiscordConnector(commands.Bot): async def get_channel_history( self, channel_id: str, - limit: int = 100, start_date: str = None, end_date: str = None, ) -> list[dict]: @@ -151,7 +150,6 @@ class DiscordConnector(commands.Bot): Args: channel_id (str): The ID of the channel to fetch messages from. - limit (int): Maximum number of messages to fetch. start_date (str): Optional start date in ISO format (YYYY-MM-DD). end_date (str): Optional end date in ISO format (YYYY-MM-DD). @@ -188,13 +186,13 @@ class DiscordConnector(commands.Bot): if end_date: try: - end_datetime = datetime.datetime.fromisoformat(f"{end_date}T23:59:59.999999").replace(tzinfo=datetime.timezone.utc) + end_datetime = datetime.datetime.fromisoformat(f"{end_date}").replace(tzinfo=datetime.timezone.utc) before = end_datetime except ValueError: logger.warning(f"Invalid end_date format: {end_date}. Ignoring.") try: - async for message in channel.history(limit=limit, before=before, after=after): + async for message in channel.history(limit=None, before=before, after=after): messages_data.append( { "id": str(message.id), diff --git a/surfsense_backend/app/tasks/connectors_indexing_tasks.py b/surfsense_backend/app/tasks/connectors_indexing_tasks.py index 0b75d16..189719e 100644 --- a/surfsense_backend/app/tasks/connectors_indexing_tasks.py +++ b/surfsense_backend/app/tasks/connectors_indexing_tasks.py @@ -993,7 +993,7 @@ async def index_discord_messages( return 0, f"Failed to get Discord guilds: {str(e)}" if not guilds: logger.info("No Discord guilds found to index") - + await discord_client.close_bot() return 0, "No Discord guilds found" @@ -1020,7 +1020,6 @@ async def index_discord_messages( channel_id=channel_id, start_date=start_date_str, end_date=end_date_str, - limit=1000 ) except Exception as e: logger.error(f"Failed to get messages for channel {channel_name}: {str(e)}")