diff --git a/crates/edit_prediction/src/fim.rs b/crates/edit_prediction/src/fim.rs
index 5482fc1fb2a..0957298c3c0 100644
--- a/crates/edit_prediction/src/fim.rs
+++ b/crates/edit_prediction/src/fim.rs
@@ -98,7 +98,10 @@ pub fn request_prediction(
         let prefix = editable_text[..cursor_in_editable].to_string();
         let suffix = editable_text[cursor_in_editable..].to_string();
         let prompt = format_fim_prompt(prompt_format, &prefix, &suffix);
-        let stop_tokens = get_fim_stop_tokens();
+        let stop_tokens = fim_stop_tokens(prompt_format)
+            .iter()
+            .map(|token| token.to_string())
+            .collect();
 
         let max_tokens = settings.max_output_tokens;
 
@@ -219,23 +222,40 @@ fn format_fim_prompt(
     }
 }
 
-fn get_fim_stop_tokens() -> Vec<String> {
-    vec![
-        "<|endoftext|>".to_string(),
-        "<|file_separator|>".to_string(),
-        "<|fim_pad|>".to_string(),
-        "<|fim_prefix|>".to_string(),
-        "<|fim_middle|>".to_string(),
-        "<|fim_suffix|>".to_string(),
-        "<fim_prefix>".to_string(),
-        "<fim_middle>".to_string(),
-        "<fim_suffix>".to_string(),
-        "<PRE>".to_string(),
-        "<SUF>".to_string(),
-        "<MID>".to_string(),
-        "[PREFIX]".to_string(),
-        "[SUFFIX]".to_string(),
-    ]
+// OpenAI-compatible servers such as vLLM reject requests with more than 4 stop
+// sequences, so each format may only list up to 4 tokens.
+fn fim_stop_tokens(prompt_format: EditPredictionPromptFormat) -> &'static [&'static str] {
+    match prompt_format {
+        EditPredictionPromptFormat::CodeLlama => &["<PRE>", "<SUF>", "<MID>"],
+        EditPredictionPromptFormat::DeepseekCoder => {
+            &["<｜fim▁begin｜>", "<｜fim▁hole｜>", "<｜fim▁end｜>"]
+        }
+        EditPredictionPromptFormat::Qwen => &[
+            "<|endoftext|>",
+            "<|fim_prefix|>",
+            "<|fim_middle|>",
+            "<|fim_suffix|>",
+        ],
+        EditPredictionPromptFormat::CodeGemma => &[
+            "<|file_separator|>",
+            "<|fim_prefix|>",
+            "<|fim_middle|>",
+            "<|fim_suffix|>",
+        ],
+        EditPredictionPromptFormat::Codestral => &["[PREFIX]", "[SUFFIX]"],
+        EditPredictionPromptFormat::Glm => &[
+            "<|endoftext|>",
+            "<|code_prefix|>",
+            "<|code_suffix|>",
+            "<|code_middle|>",
+        ],
+        _ => &[
+            "<|endoftext|>",
+            "<fim_prefix>",
+            "<fim_middle>",
+            "<fim_suffix>",
+        ],
+    }
 }
 
 fn clean_fim_completion(response: &str) -> String {
@@ -315,6 +335,46 @@ mod tests {
         );
     }
 
+    const FIM_PROMPT_FORMATS: [EditPredictionPromptFormat; 7] = [
+        EditPredictionPromptFormat::CodeLlama,
+        EditPredictionPromptFormat::StarCoder,
+        EditPredictionPromptFormat::DeepseekCoder,
+        EditPredictionPromptFormat::Qwen,
+        EditPredictionPromptFormat::CodeGemma,
+        EditPredictionPromptFormat::Codestral,
+        EditPredictionPromptFormat::Glm,
+    ];
+
+    #[test]
+    fn fim_stop_tokens_respect_openai_stop_limit() {
+        for format in FIM_PROMPT_FORMATS {
+            let stop_tokens = fim_stop_tokens(format);
+            assert!(!stop_tokens.is_empty(), "{format:?} has no stop tokens");
+            assert!(
+                stop_tokens.len() <= 4,
+                "{format:?} has {} stop tokens, but OpenAI allows at most 4",
+                stop_tokens.len()
+            );
+        }
+    }
+
+    #[test]
+    fn fim_stop_tokens_match_prompt_markers() {
+        let end_of_text_tokens = ["<|endoftext|>", "<|file_separator|>"];
+        for format in FIM_PROMPT_FORMATS {
+            let prompt = format_fim_prompt(format, "a", "b");
+            for token in fim_stop_tokens(format) {
+                if end_of_text_tokens.contains(token) {
+                    continue;
+                }
+                assert!(
+                    prompt.contains(token),
+                    "stop token {token:?} for {format:?} does not appear in prompt {prompt:?}"
+                );
+            }
+        }
+    }
+
     #[test]
     fn infer_prompt_format_returns_none_for_unsupported_models() {
         assert_eq!(infer_prompt_format("llama3:70b"), None);