vrr/ProxyAI
2
0
Fork 0
mirror of https://github.com/carlrobertoh/ProxyAI.git synced 2026-05-10 20:30:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: OpenAI and Claude vision support (#430)

Browse source 
* feat: add OpenAI and Claude vision support

* refactor: replace awaitility with PlatformTestUtil.waitWithEventsDispatching

* feat: display error when image not found

* chore: bump llm-client

* feat: configurable file watcher and minor code cleanup

* fix: ensure image notifications are triggered only for image file types

* docs: update changelog

* fix: user textarea icon button behaviour

* refactor: minor cleanup
This commit is contained in:
Carl-Robert 2024-04-02 02:50:41 +03:00 • committed by GitHub
parent 99b5f94dad
commit 8cf5720db9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
40 changed files with 793 additions and 309 deletions
Download patch file Download diff file Expand all files Collapse all files

72
src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ChatToolWindowTabPanel.java
View file

@ -6,7 +6,6 @@ import static java.lang.String.format;
import static java.util.stream.Collectors.toList;
import com.intellij.openapi.Disposable;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.editor.impl.EditorImpl;
import com.intellij.openapi.project.Project;
@ -27,7 +26,6 @@ import ee.carlrobert.codegpt.settings.GeneralSettings;
import ee.carlrobert.codegpt.settings.service.ServiceType;
import ee.carlrobert.codegpt.telemetry.TelemetryAction;
import ee.carlrobert.codegpt.toolwindow.chat.standard.StandardChatToolWindowContentManager;
import ee.carlrobert.codegpt.toolwindow.chat.standard.StandardChatToolWindowPanel;
import ee.carlrobert.codegpt.toolwindow.chat.ui.ChatMessageResponseBody;
import ee.carlrobert.codegpt.toolwindow.chat.ui.ChatToolWindowScrollablePanel;
import ee.carlrobert.codegpt.toolwindow.chat.ui.ResponsePanel;
@ -41,11 +39,15 @@ import ee.carlrobert.codegpt.util.file.FileUtil;
import java.awt.BorderLayout;
import java.awt.GridBagConstraints;
import java.awt.GridBagLayout;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.UUID;
import javax.swing.JComponent;
import javax.swing.JPanel;
import javax.swing.SwingUtilities;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public abstract class ChatToolWindowTabPanel implements Disposable {
@ -62,10 +64,7 @@ public abstract class ChatToolWindowTabPanel implements Disposable {
protected abstract JComponent getLandingView();
public ChatToolWindowTabPanel(
@NotNull Project project,
@NotNull Conversation conversation,
boolean useContextualSearch) {
public ChatToolWindowTabPanel(@NotNull Project project, @NotNull Conversation conversation) {
this.project = project;
this.conversation = conversation;
conversationService = ConversationService.getInstance();
@ -98,8 +97,10 @@ public abstract class ChatToolWindowTabPanel implements Disposable {
}
public void sendMessage(Message message, ConversationType conversationType) {
Runnable runnable = () -> {
SwingUtilities.invokeLater(() -> {
var referencedFiles = project.getUserData(CodeGPTKeys.SELECTED_FILES);
var chatToolWindowPanel = project.getService(StandardChatToolWindowContentManager.class)
.tryFindChatToolWindowPanel();
if (referencedFiles != null && !referencedFiles.isEmpty()) {
var referencedFilePaths = referencedFiles.stream()
.map(ReferencedFile::getFilePath)
@ -110,26 +111,42 @@ public abstract class ChatToolWindowTabPanel implements Disposable {
totalTokensPanel.updateReferencedFilesTokens(referencedFiles);
project.getService(StandardChatToolWindowContentManager.class)
.tryFindChatToolWindowPanel()
.ifPresent(StandardChatToolWindowPanel::clearSelectedFilesNotification);
chatToolWindowPanel.ifPresent(panel -> panel.clearNotifications(project));
}
var userMessagePanel = new UserMessagePanel(project, message, this);
var attachedFilePath = CodeGPTKeys.IMAGE_ATTACHMENT_FILE_PATH.get(project);
var callParameters = getCallParameters(conversationType, message, attachedFilePath);
if (callParameters.getImageData() != null) {
message.setImageFilePath(attachedFilePath);
chatToolWindowPanel.ifPresent(panel -> panel.clearNotifications(project));
userMessagePanel.displayImage(attachedFilePath);
}
var messagePanel = toolWindowScrollablePanel.addMessage(message.getId());
messagePanel.add(new UserMessagePanel(project, message, this));
messagePanel.add(userMessagePanel);
var responsePanel = createResponsePanel(message, conversationType);
messagePanel.add(responsePanel);
updateTotalTokens(message);
call(callParameters, responsePanel);
});
}
call(message, conversationType, responsePanel, false);
};
// TODO
if (ApplicationManager.getApplication().isUnitTestMode()) {
runnable.run();
} else {
SwingUtilities.invokeLater(runnable);
private CallParameters getCallParameters(
ConversationType conversationType,
Message message,
@Nullable String attachedFilePath) {
var callParameters = new CallParameters(conversation, conversationType, message, false);
if (attachedFilePath != null && !attachedFilePath.isEmpty()) {
try {
callParameters.setImageData(Files.readAllBytes(Path.of(attachedFilePath)));
callParameters.setImageMediaType(FileUtil.getImageMediaType(attachedFilePath));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return callParameters;
}
private void updateTotalTokens(Message message) {
@ -175,7 +192,7 @@ public abstract class ChatToolWindowTabPanel implements Disposable {
if (responsePanel != null) {
message.setResponse("");
conversationService.saveMessage(conversation, message);
call(message, conversationType, responsePanel, true);
call(new CallParameters(conversation, conversationType, message, true), responsePanel);
}
totalTokensPanel.updateConversationTokens(conversation);
@ -202,11 +219,7 @@ public abstract class ChatToolWindowTabPanel implements Disposable {
totalTokensPanel.updateConversationTokens(conversation);
}
private void call(
Message message,
ConversationType conversationType,
ResponsePanel responsePanel,
boolean retry) {
private void call(CallParameters callParameters, ResponsePanel responsePanel) {
var responseContainer = (ChatMessageResponseBody) responsePanel.getContent();
if (!CompletionRequestService.getInstance().isRequestAllowed()) {
@ -222,13 +235,13 @@ public abstract class ChatToolWindowTabPanel implements Disposable {
userPromptTextArea) {
@Override
public void handleTokensExceededPolicyAccepted() {
call(message, conversationType, responsePanel, true);
call(callParameters, responsePanel);
}
});
userPromptTextArea.setRequestHandler(requestHandler);
userPromptTextArea.setSubmitEnabled(false);
requestHandler.call(new CallParameters(conversation, conversationType, message, retry));
requestHandler.call(callParameters);
}
private void handleSubmit(String text) {
@ -257,7 +270,10 @@ public abstract class ChatToolWindowTabPanel implements Disposable {
panel.add(JBUI.Panels.simplePanel(new UserPromptTextAreaHeader(
selectedService,
totalTokensPanel,
contentManager::createNewTabPanel)), BorderLayout.NORTH);
() -> {
ConversationService.getInstance().startConversation();
contentManager.createNewTabPanel();
})), BorderLayout.NORTH);
panel.add(JBUI.Panels.simplePanel(userPromptTextArea), BorderLayout.CENTER);
return panel;
}