vrr/ruvector
2
0
Fork 0
mirror of https://github.com/ruvnet/RuVector.git synced 2026-05-27 00:25:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

fix(adr-115): add Accept and Connection headers for CDX requests

Browse source 
Try adding HTTP headers that might help with server compatibility:
- Accept: application/json
- Connection: close (avoid keep-alive issues)
This commit is contained in:
Reuven 2026-03-17 02:14:46 -04:00
parent f4a2763038
commit 4c6ea4ebcb
1 changed files with 11 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

13
crates/mcp-brain-server/src/pipeline.rs
View file

@ -644,7 +644,12 @@ impl CommonCrawlAdapter {
tokio::time::sleep(delay).await;
}
match self.http.get(&url).send().await {
// Add headers that might help with compatibility
match self.http.get(&url)
.header("Accept", "application/json")
.header("Connection", "close")
.send().await
{
Ok(resp) => {
let status = resp.status().as_u16();
match resp.text().await {
@ -744,7 +749,11 @@ impl CommonCrawlAdapter {
tokio::time::sleep(delay).await;
}
match self.http.get(&url).send().await {
match self.http.get(&url)
.header("Accept", "application/json")
.header("Connection", "close")
.send().await
{
Ok(resp) => {
if !resp.status().is_success() {
last_error = format!("CDX returned status {}", resp.status());