vikarti.anatra/Automated-AI-Web-Researcher-Ollama
1
0
Fork 0
mirror of https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama.git synced 2025-01-19 00:47:46 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Removed code to check for robots.txt

Browse source
This commit is contained in:
Graham V 2024-11-20 10:23:42 -05:00
parent 66634f4070
commit 7daf808d1d
1 changed files with 19 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
web_scraper.py
View file

@ -23,15 +23,15 @@ class WebScraper:
self.last_request_time = {} self.last_request_time = {}
def can_fetch(self, url): def can_fetch(self, url):
parsed_url = urlparse(url) # parsed_url = urlparse(url)
robots_url = f"{parsed_url.scheme}://{parsed_url.netloc}/robots.txt" # robots_url = f"{parsed_url.scheme}://{parsed_url.netloc}/robots.txt"
self.robot_parser.set_url(robots_url) # self.robot_parser.set_url(robots_url)
try: # try:
self.robot_parser.read() # self.robot_parser.read()
return self.robot_parser.can_fetch(self.session.headers["User-Agent"], url) # return self.robot_parser.can_fetch(self.session.headers["User-Agent"], url)
except Exception as e: # except Exception as e:
logger.warning(f"Error reading robots.txt for {url}: {e}") # logger.warning(f"Error reading robots.txt for {url}: {e}")
return True # Assume allowed if robots.txt can't be read return True # ignore robots.txt
def respect_rate_limit(self, url): def respect_rate_limit(self, url):
domain = urlparse(url).netloc domain = urlparse(url).netloc
@ -125,16 +125,16 @@ def get_web_content(urls):
# Standalone can_fetch function # Standalone can_fetch function
def can_fetch(url): def can_fetch(url):
parsed_url = urlparse(url) # parsed_url = urlparse(url)
robots_url = f"{parsed_url.scheme}://{parsed_url.netloc}/robots.txt" # robots_url = f"{parsed_url.scheme}://{parsed_url.netloc}/robots.txt"
rp = RobotFileParser() # rp = RobotFileParser()
rp.set_url(robots_url) # rp.set_url(robots_url)
try: # try:
rp.read() # rp.read()
return rp.can_fetch("*", url) # return rp.can_fetch("*", url)
except Exception as e: # except Exception as e:
logger.warning(f"Error reading robots.txt for {url}: {e}") # logger.warning(f"Error reading robots.txt for {url}: {e}")
return True # Assume allowed if robots.txt can't be read return True # ignore robots.xt
if __name__ == "__main__": if __name__ == "__main__":
test_urls = [ test_urls = [