diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 1d398a9..ae2ab2f 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -66,6 +66,16 @@ android:theme="@style/Theme.TransparentActivity" android:showOnLockScreen="true" /> + + + + \ No newline at end of file diff --git a/app/src/main/java/com/app/assistant/db/AppDatabase.kt b/app/src/main/java/com/app/assistant/db/AppDatabase.kt index 84b1294..9465e20 100644 --- a/app/src/main/java/com/app/assistant/db/AppDatabase.kt +++ b/app/src/main/java/com/app/assistant/db/AppDatabase.kt @@ -6,7 +6,7 @@ import androidx.room.Room import androidx.room.RoomDatabase import androidx.room.TypeConverters -@Database(entities = [GroupEntity::class, MessageEntity::class], version = 3, exportSchema = false) +@Database(entities = [GroupEntity::class, MessageEntity::class, AttachmentEntity::class], version = 4, exportSchema = false) @TypeConverters(Converters::class) abstract class AppDatabase : RoomDatabase() { abstract fun conversationDao(): ConversationDao diff --git a/app/src/main/java/com/app/assistant/db/AttachmentEntity.kt b/app/src/main/java/com/app/assistant/db/AttachmentEntity.kt new file mode 100644 index 0000000..0780a53 --- /dev/null +++ b/app/src/main/java/com/app/assistant/db/AttachmentEntity.kt @@ -0,0 +1,39 @@ +package com.app.assistant.db + +import androidx.room.ColumnInfo +import androidx.room.Entity +import androidx.room.ForeignKey +import androidx.room.Index +import androidx.room.PrimaryKey + +@Entity( + tableName = "attachments", + foreignKeys = [ + ForeignKey( + entity = MessageEntity::class, + parentColumns = ["id"], + childColumns = ["message_id"], + onDelete = ForeignKey.CASCADE + ) + ], + indices = [Index(value = ["message_id"])] +) +data class AttachmentEntity( + @PrimaryKey(autoGenerate = true) + val id: Long = 0, + + @ColumnInfo(name = "message_id") + val messageId: Long, + + @ColumnInfo(name = "file_path") + val filePath: String, + + @ColumnInfo(name = "mime_type") + val mimeType: String, + + @ColumnInfo(name = "file_name") + val fileName: String, + + @ColumnInfo(name = "iv") + val iv: String +) diff --git a/app/src/main/java/com/app/assistant/db/ConversationDao.kt b/app/src/main/java/com/app/assistant/db/ConversationDao.kt index 750c814..82b3188 100644 --- a/app/src/main/java/com/app/assistant/db/ConversationDao.kt +++ b/app/src/main/java/com/app/assistant/db/ConversationDao.kt @@ -31,4 +31,13 @@ interface ConversationDao { @Query("SELECT * FROM messages WHERE group_id = :groupId ORDER BY id ASC") suspend fun getMessagesForGroup(groupId: Long): List + + @Insert(onConflict = OnConflictStrategy.REPLACE) + suspend fun insertAttachment(attachment: AttachmentEntity) + + @Query("SELECT * FROM attachments WHERE message_id = :messageId ORDER BY id ASC") + suspend fun getAttachmentsForMessage(messageId: Long): List + + @Query("DELETE FROM attachments WHERE message_id = :messageId") + suspend fun deleteAttachmentsForMessage(messageId: Long) } diff --git a/app/src/main/java/com/app/assistant/db/DynamicConversationRepository.kt b/app/src/main/java/com/app/assistant/db/DynamicConversationRepository.kt index 43d5ef5..f756735 100644 --- a/app/src/main/java/com/app/assistant/db/DynamicConversationRepository.kt +++ b/app/src/main/java/com/app/assistant/db/DynamicConversationRepository.kt @@ -3,8 +3,11 @@ package com.app.assistant.db import android.content.Context import com.app.assistant.model.Conversation import com.app.assistant.model.Group +import com.app.assistant.model.Attachment import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import java.text.SimpleDateFormat import java.util.Date import java.util.Locale @@ -15,6 +18,7 @@ class DynamicConversationRepository( private val db = AppDatabase.getDatabase(context) private val dao = db.conversationDao() var currentGroupId: Long = -1L + private val writeMutex = Mutex() private suspend fun startNewChat(msg: String): Long { return withContext(Dispatchers.IO) { @@ -33,31 +37,60 @@ class DynamicConversationRepository( } suspend fun addMessage(conversation: Conversation) { - withContext(Dispatchers.IO) { - if (currentGroupId == -1L) { - currentGroupId = startNewChat(conversation.text) + writeMutex.withLock { + withContext(Dispatchers.IO) { + if (currentGroupId == -1L) { + currentGroupId = startNewChat(conversation.text) + } + + val iv = EncryptionUtil.generateIV() + val (encryptedText, _) = EncryptionUtil.encrypt(conversation.text, iv) + + val messageEntity = MessageEntity( + id = conversation.id, + text = encryptedText, + isMe = conversation.isMe, + category = conversation.category, + contentURL = conversation.contentURL, + navigationURI = conversation.navigationURI, + iv = iv, + groupId = currentGroupId + ) + dao.insertMessage(messageEntity) + + // Insert attachments + conversation.attachments.forEach { attachment -> + val attachmentEntity = AttachmentEntity( + id = attachment.id, + messageId = conversation.id, + filePath = attachment.filePath, + mimeType = attachment.mimeType, + fileName = attachment.fileName, + iv = attachment.iv + ) + dao.insertAttachment(attachmentEntity) + } } - - val iv = EncryptionUtil.generateIV() - val (encryptedText, _) = EncryptionUtil.encrypt(conversation.text, iv) - - val messageEntity = MessageEntity( - id = conversation.id, - text = encryptedText, - isMe = conversation.isMe, - category = conversation.category, - contentURL = conversation.contentURL, - navigationURI = conversation.navigationURI, - iv = iv, - groupId = currentGroupId - ) - dao.insertMessage(messageEntity) } } suspend fun deleteMessage(id: Long) { - withContext(Dispatchers.IO) { - dao.deleteMessageById(id) + writeMutex.withLock { + withContext(Dispatchers.IO) { + val attachmentEntities = dao.getAttachmentsForMessage(id) + attachmentEntities.forEach { att -> + try { + val file = java.io.File(att.filePath) + if (file.exists()) { + file.delete() + } + } catch (e: Exception) { + e.printStackTrace() + } + } + dao.deleteAttachmentsForMessage(id) + dao.deleteMessageById(id) + } } } @@ -65,30 +98,62 @@ class DynamicConversationRepository( oldConversation: Conversation, newConversation: Conversation, ) { - withContext(Dispatchers.IO) { - val iv = EncryptionUtil.generateIV() - val (encryptedText, _) = EncryptionUtil.encrypt(newConversation.text, iv) + writeMutex.withLock { + withContext(Dispatchers.IO) { + val iv = EncryptionUtil.generateIV() + val (encryptedText, _) = EncryptionUtil.encrypt(newConversation.text, iv) - val messageEntity = MessageEntity( - id = newConversation.id, - text = encryptedText, - isMe = newConversation.isMe, - category = newConversation.category, - contentURL = newConversation.contentURL, - navigationURI = newConversation.navigationURI, - iv = iv, - groupId = currentGroupId - ) - dao.updateMessage(messageEntity) + val messageEntity = MessageEntity( + id = newConversation.id, + text = encryptedText, + isMe = newConversation.isMe, + category = newConversation.category, + contentURL = newConversation.contentURL, + navigationURI = newConversation.navigationURI, + iv = iv, + groupId = currentGroupId + ) + dao.updateMessage(messageEntity) + + // Delete old attachments first and insert new ones + dao.deleteAttachmentsForMessage(newConversation.id) + newConversation.attachments.forEach { attachment -> + val attachmentEntity = AttachmentEntity( + id = attachment.id, + messageId = newConversation.id, + filePath = attachment.filePath, + mimeType = attachment.mimeType, + fileName = attachment.fileName, + iv = attachment.iv + ) + dao.insertAttachment(attachmentEntity) + } + } } } suspend fun clearMessages(conversations: List) { - withContext(Dispatchers.IO) { - val ids = conversations.map { it.id } - dao.deleteMessagesByIds(ids) - dao.deleteGroupById(currentGroupId) - currentGroupId = -1L + writeMutex.withLock { + withContext(Dispatchers.IO) { + conversations.forEach { conversation -> + val attachmentEntities = dao.getAttachmentsForMessage(conversation.id) + attachmentEntities.forEach { att -> + try { + val file = java.io.File(att.filePath) + if (file.exists()) { + file.delete() + } + } catch (e: Exception) { + e.printStackTrace() + } + } + dao.deleteAttachmentsForMessage(conversation.id) + } + val ids = conversations.map { it.id } + dao.deleteMessagesByIds(ids) + dao.deleteGroupById(currentGroupId) + currentGroupId = -1L + } } } @@ -104,6 +169,16 @@ class DynamicConversationRepository( return withContext(Dispatchers.IO) { dao.getMessagesForGroup(groupId).map { entity -> val decryptedText = EncryptionUtil.decrypt(entity.text, entity.iv) + val attachmentEntities = dao.getAttachmentsForMessage(entity.id) + val attachments = attachmentEntities.map { attEntity -> + Attachment( + id = attEntity.id, + filePath = attEntity.filePath, + mimeType = attEntity.mimeType, + fileName = attEntity.fileName, + iv = attEntity.iv + ) + } Conversation( id = entity.id, @@ -112,7 +187,8 @@ class DynamicConversationRepository( isLoading = false, category = entity.category, contentURL = entity.contentURL, - navigationURI = entity.navigationURI + navigationURI = entity.navigationURI, + attachments = attachments ) }.toMutableList() } diff --git a/app/src/main/java/com/app/assistant/db/EncryptionUtil.kt b/app/src/main/java/com/app/assistant/db/EncryptionUtil.kt index 15c8302..c305f34 100644 --- a/app/src/main/java/com/app/assistant/db/EncryptionUtil.kt +++ b/app/src/main/java/com/app/assistant/db/EncryptionUtil.kt @@ -75,6 +75,35 @@ object EncryptionUtil { } } + // Encrypt a file byte array using IV + fun encryptFile(inputBytes: ByteArray, ivString: String): ByteArray { + if (inputBytes.isEmpty()) return ByteArray(0) + return try { + val iv = Base64.decode(ivString, Base64.DEFAULT) + val cipher = Cipher.getInstance(TRANSFORMATION) + val spec = GCMParameterSpec(TAG_LENGTH_BIT, iv) + cipher.init(Cipher.ENCRYPT_MODE, secretKey, spec) + cipher.doFinal(inputBytes) + } catch (e: Exception) { + inputBytes + } + } + + // Decrypt a file byte array using IV + fun decryptFile(encryptedBytes: ByteArray, ivString: String): ByteArray { + if (encryptedBytes.isEmpty() || ivString.isEmpty()) return encryptedBytes + return try { + val iv = Base64.decode(ivString, Base64.DEFAULT) + val cipher = Cipher.getInstance(TRANSFORMATION) + val spec = GCMParameterSpec(TAG_LENGTH_BIT, iv) + cipher.init(Cipher.DECRYPT_MODE, secretKey, spec) + cipher.doFinal(encryptedBytes) + } catch (e: Exception) { + encryptedBytes + } + } + + // Retrieve or create a persistent AES key in Android KeyStore private fun getOrCreateSecretKey(): SecretKey { return try { diff --git a/app/src/main/java/com/app/assistant/llm/FlexibleLlmAdapter.kt b/app/src/main/java/com/app/assistant/llm/FlexibleLlmAdapter.kt index 51f0138..4c3b82f 100644 --- a/app/src/main/java/com/app/assistant/llm/FlexibleLlmAdapter.kt +++ b/app/src/main/java/com/app/assistant/llm/FlexibleLlmAdapter.kt @@ -18,7 +18,8 @@ class FlexibleLlmAdapter( private val client: OkHttpClient, private val apiKey: String, private val model: String, - private val config: LlmConfig + private val config: LlmConfig, + private val provider: LlmProvider ) : LlmAdapter { private val json = Json { ignoreUnknownKeys = true } @@ -174,10 +175,14 @@ class FlexibleLlmAdapter( "assistant" -> config.assistantRole else -> msg.role } - val escapedContent = escapeJsonString(msg.content) - val msgJson = config.messageFormat - .replace("{{ROLE}}", roleValue) - .replace("{{CONTENT}}", escapedContent) + val msgJson = if (msg.attachments.isNullOrEmpty()) { + val escapedContent = escapeJsonString(msg.content) + config.messageFormat + .replace("{{ROLE}}", roleValue) + .replace("{{CONTENT}}", escapedContent) + } else { + buildMultimodalMessageJson(roleValue, msg.content, msg.attachments, provider) + } serializedMessagesBuilder.append(msgJson) if (index < messagesToSerialize.size - 1) { @@ -205,6 +210,46 @@ class FlexibleLlmAdapter( return requestBodyString } + private fun buildMultimodalMessageJson( + role: String, + content: String, + attachments: List, + provider: LlmProvider + ): String { + return when (provider) { + LlmProvider.GEMINI -> { + val parts = mutableListOf() + parts.add("{\"text\": \"${escapeJsonString(content)}\"}") + attachments.forEach { att -> + parts.add("{\"inlineData\": {\"mimeType\": \"${att.mimeType}\", \"data\": \"${att.base64Data}\"}}") + } + "{\"role\": \"$role\", \"parts\": [${parts.joinToString(",")}]}" + } + LlmProvider.ANTHROPIC -> { + val contentArray = mutableListOf() + contentArray.add("{\"type\": \"text\", \"text\": \"${escapeJsonString(content)}\"}") + attachments.forEach { att -> + if (att.mimeType.startsWith("image/")) { + contentArray.add("{\"type\": \"image\", \"source\": {\"type\": \"base64\", \"media_type\": \"${att.mimeType}\", \"data\": \"${att.base64Data}\"}}") + } else if (att.mimeType == "application/pdf") { + contentArray.add("{\"type\": \"document\", \"source\": {\"type\": \"base64\", \"media_type\": \"${att.mimeType}\", \"data\": \"${att.base64Data}\"}}") + } + } + "{\"role\": \"$role\", \"content\": [${contentArray.joinToString(",")}]}" + } + else -> { // OpenAI, Groq, Ollama, DeepSeek, OpenRouter, Custom + val contentArray = mutableListOf() + contentArray.add("{\"type\": \"text\", \"text\": \"${escapeJsonString(content)}\"}") + attachments.forEach { att -> + if (att.mimeType.startsWith("image/")) { + contentArray.add("{\"type\": \"image_url\", \"image_url\": {\"url\": \"data:${att.mimeType};base64,${att.base64Data}\"}}") + } + } + "{\"role\": \"$role\", \"content\": [${contentArray.joinToString(",")}]}" + } + } + } + private fun extractChunkContent(jsonStr: String, path: String): String? { var extracted = extractValueFromJson(jsonStr, path) if (extracted != null) return extracted diff --git a/app/src/main/java/com/app/assistant/llm/LlmMessage.kt b/app/src/main/java/com/app/assistant/llm/LlmMessage.kt index a40f507..58e5362 100644 --- a/app/src/main/java/com/app/assistant/llm/LlmMessage.kt +++ b/app/src/main/java/com/app/assistant/llm/LlmMessage.kt @@ -2,8 +2,16 @@ package com.app.assistant.llm import kotlinx.serialization.Serializable +@Serializable +data class LlmAttachment( + val base64Data: String, + val mimeType: String, + val fileName: String +) + @Serializable data class LlmMessage( val role: String, - val content: String + val content: String, + val attachments: List? = null ) diff --git a/app/src/main/java/com/app/assistant/llm/ModelCapabilities.kt b/app/src/main/java/com/app/assistant/llm/ModelCapabilities.kt index 8662aa6..d51c966 100644 --- a/app/src/main/java/com/app/assistant/llm/ModelCapabilities.kt +++ b/app/src/main/java/com/app/assistant/llm/ModelCapabilities.kt @@ -3,5 +3,6 @@ package com.app.assistant.llm data class ModelCapabilities( val hasImageInput: Boolean, val hasAudioInput: Boolean, - val hasVideoInput: Boolean + val hasVideoInput: Boolean, + val hasDocumentInput: Boolean ) diff --git a/app/src/main/java/com/app/assistant/llm/ModelCapabilityProber.kt b/app/src/main/java/com/app/assistant/llm/ModelCapabilityProber.kt index 7544055..b7600c4 100644 --- a/app/src/main/java/com/app/assistant/llm/ModelCapabilityProber.kt +++ b/app/src/main/java/com/app/assistant/llm/ModelCapabilityProber.kt @@ -19,11 +19,12 @@ object ModelCapabilityProber { val model = modelName.lowercase() return when (provider) { LlmProvider.GEMINI -> { - val isMultimodal = model.contains("gemini-1.5") || model.contains("gemini-2.0") || model.contains("gemini-pro") + val isMultimodal = model.contains("gemini-1.5") || model.contains("gemini-2.0") || model.contains("gemini-pro") || model.contains("gemini-2.5") ModelCapabilities( hasImageInput = isMultimodal || model.contains("vision"), hasAudioInput = isMultimodal, - hasVideoInput = isMultimodal + hasVideoInput = isMultimodal, + hasDocumentInput = isMultimodal ) } LlmProvider.OPENAI -> { @@ -32,7 +33,8 @@ object ModelCapabilityProber { ModelCapabilities( hasImageInput = hasVision, hasAudioInput = hasAudio, - hasVideoInput = false + hasVideoInput = false, + hasDocumentInput = false ) } LlmProvider.ANTHROPIC -> { @@ -40,14 +42,16 @@ object ModelCapabilityProber { ModelCapabilities( hasImageInput = supportsVision, hasAudioInput = false, - hasVideoInput = false + hasVideoInput = false, + hasDocumentInput = supportsVision ) } LlmProvider.GROQ -> { ModelCapabilities( hasImageInput = model.contains("vision") || model.contains("scout"), hasAudioInput = false, - hasVideoInput = false + hasVideoInput = false, + hasDocumentInput = false ) } LlmProvider.OLLAMA -> { @@ -55,31 +59,36 @@ object ModelCapabilityProber { ModelCapabilities( hasImageInput = isMultimodal, hasAudioInput = false, - hasVideoInput = false + hasVideoInput = false, + hasDocumentInput = false ) } LlmProvider.DEEPSEEK -> { ModelCapabilities( hasImageInput = model.contains("vl"), hasAudioInput = false, - hasVideoInput = false + hasVideoInput = false, + hasDocumentInput = false ) } LlmProvider.OPEN_ROUTER -> { val hasVision = model.contains("vision") || model.contains("llava") || model.contains("gpt-4o") || model.contains("claude-3") || model.contains("gemini") val hasAudio = model.contains("gemini") || model.contains("audio") val hasVideo = model.contains("gemini") + val hasDoc = model.contains("gemini") || model.contains("claude-3") ModelCapabilities( hasImageInput = hasVision, hasAudioInput = hasAudio, - hasVideoInput = hasVideo + hasVideoInput = hasVideo, + hasDocumentInput = hasDoc ) } LlmProvider.CUSTOM -> { ModelCapabilities( hasImageInput = false, hasAudioInput = false, - hasVideoInput = false + hasVideoInput = false, + hasDocumentInput = false ) } } diff --git a/app/src/main/java/com/app/assistant/model/Attachment.kt b/app/src/main/java/com/app/assistant/model/Attachment.kt new file mode 100644 index 0000000..91504bb --- /dev/null +++ b/app/src/main/java/com/app/assistant/model/Attachment.kt @@ -0,0 +1,11 @@ +package com.app.assistant.model + +import java.io.Serializable + +data class Attachment( + val id: Long = System.currentTimeMillis() * 1_000_000 + (System.nanoTime() % 1_000_000), + val filePath: String, + val mimeType: String, + val fileName: String, + val iv: String +) : Serializable diff --git a/app/src/main/java/com/app/assistant/model/Conversation.kt b/app/src/main/java/com/app/assistant/model/Conversation.kt index a5651c6..cce4665 100644 --- a/app/src/main/java/com/app/assistant/model/Conversation.kt +++ b/app/src/main/java/com/app/assistant/model/Conversation.kt @@ -11,6 +11,7 @@ data class Conversation( val contentURL: String = "", val navigationURI: URI = URI(""), val isStreaming: Boolean = false, + val attachments: List = emptyList(), ) { fun getThinkingProcess(): String? { val start = text.indexOf("") @@ -25,10 +26,17 @@ data class Conversation( fun getActualAnswer(): String { val end = text.indexOf("") - return if (end == -1) { + val mainText = if (end == -1) { if (text.contains("")) "" else text } else { text.substring(end + 8) } + if (isMe) { + val fileIndex = mainText.indexOf("\n\n[Attached File:") + if (fileIndex != -1) { + return mainText.substring(0, fileIndex) + } + } + return mainText } } diff --git a/app/src/main/java/com/app/assistant/repository/SettingsRepository.kt b/app/src/main/java/com/app/assistant/repository/SettingsRepository.kt index 7c4f657..ad4b67f 100644 --- a/app/src/main/java/com/app/assistant/repository/SettingsRepository.kt +++ b/app/src/main/java/com/app/assistant/repository/SettingsRepository.kt @@ -1,4 +1,4 @@ -package com.app.assistant.repository +package com.app.assistant.repository import android.content.Context import android.content.SharedPreferences @@ -126,6 +126,11 @@ class SettingsRepository( sharedPreferences.edit().putBoolean("is_video_supported", supported).apply() } + fun getIsDocumentSupported(): Boolean = sharedPreferences.getBoolean("is_document_supported", false) + fun setIsDocumentSupported(supported: Boolean) { + sharedPreferences.edit().putBoolean("is_document_supported", supported).apply() + } + fun getIsModelVerified(): Boolean = sharedPreferences.getBoolean("is_model_verified", false) fun setIsModelVerified(verified: Boolean) { sharedPreferences.edit().putBoolean("is_model_verified", verified).apply() diff --git a/app/src/main/java/com/app/assistant/ui/screen/ChatScreen.kt b/app/src/main/java/com/app/assistant/ui/screen/ChatScreen.kt index 96735ed..5c89605 100644 --- a/app/src/main/java/com/app/assistant/ui/screen/ChatScreen.kt +++ b/app/src/main/java/com/app/assistant/ui/screen/ChatScreen.kt @@ -111,6 +111,29 @@ fun SetupUI(viewModel: MainViewModel, settingsViewModel: SettingsViewModel) { var currentScreen by remember { mutableStateOf("chat") } + val selectedAttachments by viewModel.selectedAttachments.collectAsState() + val isImageSupported = remember(currentScreen) { viewModel.settingsRepository.getIsImageSupported() } + val isAudioSupported = remember(currentScreen) { viewModel.settingsRepository.getIsAudioSupported() } + val isVideoSupported = remember(currentScreen) { viewModel.settingsRepository.getIsVideoSupported() } + val isDocumentSupported = remember(currentScreen) { viewModel.settingsRepository.getIsDocumentSupported() } + + val pickerLauncher = androidx.activity.compose.rememberLauncherForActivityResult( + contract = androidx.activity.result.contract.ActivityResultContracts.GetContent() + ) { uri -> + uri?.let { viewModel.addSelectedAttachment(it) } + } + + var tempCameraUri by remember { mutableStateOf(null) } + val cameraLauncher = androidx.activity.compose.rememberLauncherForActivityResult( + contract = androidx.activity.result.contract.ActivityResultContracts.TakePicture() + ) { success -> + if (success) { + tempCameraUri?.let { uri -> + viewModel.addSelectedAttachment(uri) + } + } + } + val context = LocalContext.current LaunchedEffect(Unit) { viewModel.showToastEvent.collectLatest { message: String -> @@ -190,7 +213,29 @@ fun SetupUI(viewModel: MainViewModel, settingsViewModel: SettingsViewModel) { onTranslationEnabledChange = { viewModel.updateTranslationEnabled(it) }, onLanguageSelected = { viewModel.onItemSelected(it) }, onDismissBottomSheet = { viewModel.setShowBottomSheet(false) }, - onBackPressed = { /* Handled in parent context back handler */ } + onBackPressed = { /* Handled in parent context back handler */ }, + selectedAttachments = selectedAttachments, + onRemoveAttachment = { viewModel.removeSelectedAttachment(it) }, + onAttachClick = { type -> + if (type == "camera") { + try { + val file = java.io.File(context.cacheDir, "camera_capture_${System.currentTimeMillis()}.jpg") + val authority = "${context.packageName}.fileprovider" + val uri = androidx.core.content.FileProvider.getUriForFile(context, authority, file) + tempCameraUri = uri + cameraLauncher.launch(uri) + } catch (e: Exception) { + android.util.Log.e("ChatScreen", "Failed to launch camera", e) + Toast.makeText(context, "Failed to launch camera", Toast.LENGTH_SHORT).show() + } + } else { + pickerLauncher.launch(type) + } + }, + isImageSupported = isImageSupported, + isAudioSupported = isAudioSupported, + isVideoSupported = isVideoSupported, + isDocumentSupported = isDocumentSupported ) } } @@ -227,6 +272,13 @@ fun ChatScreenContent( onLanguageSelected: (String) -> Unit, onDismissBottomSheet: () -> Unit, onBackPressed: () -> Unit, + selectedAttachments: List = emptyList(), + onRemoveAttachment: (com.app.assistant.model.Attachment) -> Unit = {}, + onAttachClick: (String) -> Unit = {}, + isImageSupported: Boolean = false, + isAudioSupported: Boolean = false, + isVideoSupported: Boolean = false, + isDocumentSupported: Boolean = false, ) { var showCopyIcon by remember { mutableStateOf(false) } var selectedItemIndex by remember { mutableStateOf(null) } @@ -332,6 +384,13 @@ fun ChatScreenContent( onStartListening = onStartListening, onProcessQuestion = onProcessQuestion, isTranslateEnabled = isTranslationEnabled, + selectedAttachments = selectedAttachments, + onRemoveAttachment = onRemoveAttachment, + onAttachClick = onAttachClick, + isImageSupported = isImageSupported, + isAudioSupported = isAudioSupported, + isVideoSupported = isVideoSupported, + isDocumentSupported = isDocumentSupported, ) } } diff --git a/app/src/main/java/com/app/assistant/ui/screen/ConversationItem.kt b/app/src/main/java/com/app/assistant/ui/screen/ConversationItem.kt index ac12b88..bbd28a9 100644 --- a/app/src/main/java/com/app/assistant/ui/screen/ConversationItem.kt +++ b/app/src/main/java/com/app/assistant/ui/screen/ConversationItem.kt @@ -39,11 +39,14 @@ import androidx.compose.ui.res.stringResource import coil3.compose.AsyncImage import com.app.assistant.R import com.app.assistant.model.Conversation +import com.app.assistant.model.Attachment import com.app.assistant.util.Category import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.unit.sp import com.app.assistant.ui.theme.AssistantTheme import androidx.compose.animation.AnimatedVisibility +import androidx.compose.animation.core.animateFloatAsState +import androidx.compose.animation.core.tween import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.size import androidx.compose.material.icons.filled.KeyboardArrowDown @@ -53,6 +56,20 @@ import androidx.compose.runtime.setValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.ui.text.font.FontWeight +import androidx.compose.foundation.Image +import androidx.compose.ui.graphics.asImageBitmap +import androidx.compose.ui.layout.ContentScale +import androidx.compose.runtime.rememberCoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.coroutines.launch +import kotlinx.coroutines.delay +import androidx.compose.ui.platform.LocalContext +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.ui.draw.clip +import androidx.compose.material3.IconButton +import androidx.compose.ui.res.painterResource @Composable @Suppress("ktlint:standard:function-naming") @@ -103,7 +120,7 @@ fun ConversationItem( LoadingDots() // Show loading animation when isLoading is true } else { if (conversation.isMe) { - OtherCard(conversation, onLongClick, index) + OtherCard(conversation, onLongClick, index, cornerRadius) } else { when (conversation.category) { Category.CALL.name -> { @@ -111,12 +128,12 @@ fun ConversationItem( } Category.OTHER.name -> { - OtherCard(conversation, onLongClick, index) + OtherCard(conversation, onLongClick, index, cornerRadius) } Category.SETTINGS.name -> { // TODO - OtherCard(conversation, onLongClick, index) + OtherCard(conversation, onLongClick, index, cornerRadius) } Category.SONGS.name -> { @@ -132,15 +149,15 @@ fun ConversationItem( } Category.REMINDER.name -> { - OtherCard(conversation, onLongClick, index) + OtherCard(conversation, onLongClick, index, cornerRadius) } Category.ALARM.name -> { - OtherCard(conversation, onLongClick, index) + OtherCard(conversation, onLongClick, index, cornerRadius) } else -> { - OtherCard(conversation, onLongClick, index) + OtherCard(conversation, onLongClick, index, cornerRadius) } } } @@ -279,13 +296,14 @@ private fun OtherCard( conversation: Conversation, onLongClick: (Int) -> Unit, index: Int, + cornerRadius: Dp = 20.dp, ) { val thinkingText = conversation.getThinkingProcess() val answerText = conversation.getActualAnswer() + val hasText = answerText.isNotEmpty() || conversation.isStreaming || thinkingText != null Column( modifier = Modifier - .padding(16.dp, 8.dp) .pointerInput(Unit) { detectTapGestures( onLongPress = { @@ -294,77 +312,504 @@ private fun OtherCard( ) } ) { - if (thinkingText != null) { - var isExpanded by remember { mutableStateOf(true) } - - Card( + if (conversation.attachments.isNotEmpty()) { + ConversationAttachments( + attachments = conversation.attachments, + hasTextBelow = hasText, + cornerRadius = cornerRadius, modifier = Modifier - .fillMaxWidth() - .padding(bottom = 8.dp), - shape = RoundedCornerShape(12.dp), - colors = CardDefaults.cardColors( - containerColor = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.5f) + ) + } + + if (thinkingText != null || answerText.isNotEmpty() || conversation.isStreaming) { + Column( + modifier = Modifier.padding( + start = 16.dp, + end = 16.dp, + top = if (conversation.attachments.isNotEmpty()) 8.dp else 8.dp, + bottom = 8.dp ) ) { - Column(modifier = Modifier.padding(12.dp)) { - Row( + if (thinkingText != null) { + var isExpanded by remember { mutableStateOf(true) } + + Card( modifier = Modifier .fillMaxWidth() - .clickable { isExpanded = !isExpanded }, - verticalAlignment = Alignment.CenterVertically + .padding(bottom = 8.dp), + shape = RoundedCornerShape(12.dp), + colors = CardDefaults.cardColors( + containerColor = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.5f) + ) ) { - Text( - text = "🧠", - fontSize = 16.sp, - modifier = Modifier.padding(end = 8.dp) - ) - Text( - text = "Thinking Process", - style = MaterialTheme.typography.labelLarge.copy( - fontWeight = FontWeight.Bold, - color = MaterialTheme.colorScheme.primary - ), - modifier = Modifier.weight(1f) - ) - Icon( - imageVector = if (isExpanded) { - Icons.Default.KeyboardArrowUp - } else { - Icons.Default.KeyboardArrowDown - }, - contentDescription = if (isExpanded) "Collapse" else "Expand", - tint = MaterialTheme.colorScheme.primary, - modifier = Modifier.size(20.dp) - ) + Column(modifier = Modifier.padding(12.dp)) { + Row( + modifier = Modifier + .fillMaxWidth() + .clickable { isExpanded = !isExpanded }, + verticalAlignment = Alignment.CenterVertically + ) { + Text( + text = "🧠", + fontSize = 16.sp, + modifier = Modifier.padding(end = 8.dp) + ) + Text( + text = "Thinking Process", + style = MaterialTheme.typography.labelLarge.copy( + fontWeight = FontWeight.Bold, + color = MaterialTheme.colorScheme.primary + ), + modifier = Modifier.weight(1f) + ) + Icon( + imageVector = if (isExpanded) { + Icons.Default.KeyboardArrowUp + } else { + Icons.Default.KeyboardArrowDown + }, + contentDescription = if (isExpanded) "Collapse" else "Expand", + tint = MaterialTheme.colorScheme.primary, + modifier = Modifier.size(20.dp) + ) + } + + AnimatedVisibility(visible = isExpanded) { + Column { + Spacer(modifier = Modifier.height(8.dp)) + Text( + text = thinkingText, + style = MaterialTheme.typography.bodyMedium.copy( + color = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.8f) + ), + modifier = Modifier.fillMaxWidth() + ) + } + } + } + } + } + + if (answerText.isNotEmpty() || conversation.isStreaming) { + val displayAnswer = if (conversation.isStreaming) { + "$answerText ▮" + } else { + answerText } - AnimatedVisibility(visible = isExpanded) { - Column { - Spacer(modifier = Modifier.height(8.dp)) - Text( - text = thinkingText, - style = MaterialTheme.typography.bodyMedium.copy( - color = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.8f) - ), - modifier = Modifier.fillMaxWidth() - ) - } + MarkdownText( + markdown = displayAnswer, + modifier = Modifier + ) + } + } + } + } +} + +@Composable +fun ConversationAttachments( + attachments: List, + hasTextBelow: Boolean, + cornerRadius: Dp, + modifier: Modifier = Modifier +) { + Column( + modifier = modifier.fillMaxWidth(), + verticalArrangement = Arrangement.spacedBy(8.dp) + ) { + attachments.forEachIndexed { index, attachment -> + val isFirst = index == 0 + val isLast = index == attachments.size - 1 + when { + attachment.mimeType.startsWith("image/") -> { + ImageAttachmentBubble( + attachment = attachment, + isFirst = isFirst, + isLast = isLast, + hasTextBelow = hasTextBelow, + cornerRadius = cornerRadius + ) + } + attachment.mimeType.startsWith("audio/") -> { + Box(modifier = Modifier.padding(horizontal = 16.dp, vertical = 4.dp)) { + AudioAttachmentBubble(attachment) + } + } + attachment.mimeType.startsWith("video/") -> { + Box(modifier = Modifier.padding(horizontal = 16.dp, vertical = 4.dp)) { + VideoAttachmentBubble(attachment) + } + } + else -> { + Box(modifier = Modifier.padding(horizontal = 16.dp, vertical = 4.dp)) { + DocumentAttachmentBubble(attachment) } } } } + } +} - if (answerText.isNotEmpty() || conversation.isStreaming) { - val displayAnswer = if (conversation.isStreaming) { - "$answerText ▮" - } else { - answerText +@Composable +fun ImageAttachmentBubble( + attachment: Attachment, + isFirst: Boolean, + isLast: Boolean, + hasTextBelow: Boolean, + cornerRadius: Dp +) { + val context = LocalContext.current + val coroutineScope = rememberCoroutineScope() + var bitmap by remember { mutableStateOf(null) } + var isLoading by remember { mutableStateOf(true) } + var imageLoaded by remember { mutableStateOf(false) } + + LaunchedEffect(attachment) { + isLoading = true + imageLoaded = false + withContext(Dispatchers.IO) { + try { + val file = java.io.File(attachment.filePath) + if (file.exists()) { + val fileBytes = file.readBytes() + val decrypted = com.app.assistant.db.EncryptionUtil.decryptFile(fileBytes, attachment.iv) + val decoded = android.graphics.BitmapFactory.decodeByteArray(decrypted, 0, decrypted.size) + bitmap = decoded + if (decoded != null) { + imageLoaded = true + } + } + } catch (e: Exception) { + e.printStackTrace() + } finally { + isLoading = false } + } + } - MarkdownText( - markdown = displayAnswer, + val topRounding = if (isFirst) cornerRadius else 0.dp + val bottomRounding = if (isLast && !hasTextBelow) cornerRadius else 0.dp + val imageShape = RoundedCornerShape( + topStart = topRounding, + topEnd = topRounding, + bottomStart = bottomRounding, + bottomEnd = bottomRounding + ) + + val imageAlpha by animateFloatAsState( + targetValue = if (imageLoaded) 1f else 0f, + animationSpec = tween(durationMillis = 400), + label = "imageFadeIn" + ) + + Box( + modifier = Modifier + .fillMaxWidth() + .clip(imageShape) + .background(MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.3f)) + .clickable { + coroutineScope.launch(Dispatchers.IO) { + try { + val fileBytes = java.io.File(attachment.filePath).readBytes() + val decrypted = com.app.assistant.db.EncryptionUtil.decryptFile(fileBytes, attachment.iv) + val ext = attachment.fileName.substringAfterLast('.', "jpg") + val temp = java.io.File(context.cacheDir, "view_${attachment.id}.$ext") + temp.writeBytes(decrypted) + + val uri = androidx.core.content.FileProvider.getUriForFile( + context, + "com.app.assistant.fileprovider", + temp + ) + val intent = android.content.Intent(android.content.Intent.ACTION_VIEW).apply { + setDataAndType(uri, "image/*") + addFlags(android.content.Intent.FLAG_GRANT_READ_URI_PERMISSION) + } + context.startActivity(intent) + } catch (e: Exception) { + withContext(Dispatchers.Main) { + android.widget.Toast.makeText(context, "Cannot open image: ${e.message}", android.widget.Toast.LENGTH_SHORT).show() + } + } + } + }, + contentAlignment = Alignment.Center + ) { + if (isLoading) { + Box( modifier = Modifier + .fillMaxWidth() + .height(240.dp), + contentAlignment = Alignment.Center + ) { + CircularProgressIndicator( + modifier = Modifier.size(36.dp), + color = MaterialTheme.colorScheme.primary, + strokeWidth = 3.dp + ) + } + } else if (bitmap != null) { + Image( + bitmap = bitmap!!.asImageBitmap(), + contentDescription = null, + modifier = Modifier + .fillMaxWidth() + .height(240.dp), + contentScale = ContentScale.Crop, + alpha = imageAlpha ) + } else { + Box( + modifier = Modifier + .fillMaxWidth() + .height(100.dp), + contentAlignment = Alignment.Center + ) { + Text( + text = "Failed to load image", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.error + ) + } + } + } +} + +@Composable +fun AudioAttachmentBubble(attachment: Attachment) { + val context = LocalContext.current + var isPlaying by remember { mutableStateOf(false) } + var progress by remember { mutableStateOf(0f) } + var duration by remember { mutableStateOf(0) } + var currentPosition by remember { mutableStateOf(0) } + val mediaPlayer = remember { android.media.MediaPlayer() } + var tempFile by remember { mutableStateOf(null) } + val coroutineScope = rememberCoroutineScope() + + androidx.compose.runtime.DisposableEffect(Unit) { + onDispose { + try { + if (mediaPlayer.isPlaying) { + mediaPlayer.stop() + } + mediaPlayer.release() + } catch (e: Exception) {} + tempFile?.delete() + } + } + + LaunchedEffect(isPlaying) { + if (isPlaying) { + while (isPlaying) { + try { + currentPosition = mediaPlayer.currentPosition + progress = if (duration > 0) currentPosition.toFloat() / duration else 0f + } catch (e: Exception) {} + kotlinx.coroutines.delay(200) + } + } + } + + Card( + colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.5f)), + shape = RoundedCornerShape(12.dp), + modifier = Modifier.fillMaxWidth().padding(vertical = 4.dp) + ) { + Row( + modifier = Modifier.padding(8.dp), + verticalAlignment = Alignment.CenterVertically + ) { + IconButton(onClick = { + if (isPlaying) { + mediaPlayer.pause() + isPlaying = false + } else { + if (tempFile == null) { + coroutineScope.launch(Dispatchers.IO) { + try { + val fileBytes = java.io.File(attachment.filePath).readBytes() + val decrypted = com.app.assistant.db.EncryptionUtil.decryptFile(fileBytes, attachment.iv) + val temp = java.io.File(context.cacheDir, "play_${attachment.id}.mp3") + temp.writeBytes(decrypted) + tempFile = temp + + withContext(Dispatchers.Main) { + mediaPlayer.reset() + mediaPlayer.setDataSource(temp.absolutePath) + mediaPlayer.prepare() + duration = mediaPlayer.duration + mediaPlayer.start() + isPlaying = true + mediaPlayer.setOnCompletionListener { + isPlaying = false + progress = 0f + currentPosition = 0 + } + } + } catch (e: Exception) { + e.printStackTrace() + } + } + } else { + mediaPlayer.start() + isPlaying = true + } + } + }) { + Icon( + painter = painterResource(id = if (isPlaying) R.drawable.ic_stop else R.drawable.ic_mic), + contentDescription = if (isPlaying) "Pause" else "Play", + tint = MaterialTheme.colorScheme.primary + ) + } + Spacer(modifier = Modifier.width(8.dp)) + Column(modifier = Modifier.weight(1f)) { + Text( + text = attachment.fileName, + style = MaterialTheme.typography.labelMedium, + maxLines = 1, + overflow = TextOverflow.Ellipsis + ) + Spacer(modifier = Modifier.height(4.dp)) + androidx.compose.material3.LinearProgressIndicator( + progress = { progress }, + modifier = Modifier.fillMaxWidth().height(4.dp), + color = MaterialTheme.colorScheme.primary, + trackColor = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.2f) + ) + } + } + } +} + +@Composable +fun VideoAttachmentBubble(attachment: Attachment) { + val context = LocalContext.current + val coroutineScope = rememberCoroutineScope() + Card( + colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.5f)), + shape = RoundedCornerShape(12.dp), + modifier = Modifier.fillMaxWidth().padding(vertical = 4.dp).clickable { + coroutineScope.launch(Dispatchers.IO) { + try { + val fileBytes = java.io.File(attachment.filePath).readBytes() + val decrypted = com.app.assistant.db.EncryptionUtil.decryptFile(fileBytes, attachment.iv) + val ext = attachment.fileName.substringAfterLast('.', "mp4") + val temp = java.io.File(context.cacheDir, "play_${attachment.id}.$ext") + temp.writeBytes(decrypted) + + val uri = androidx.core.content.FileProvider.getUriForFile( + context, + "com.app.assistant.fileprovider", + temp + ) + val intent = android.content.Intent(android.content.Intent.ACTION_VIEW).apply { + setDataAndType(uri, "video/*") + addFlags(android.content.Intent.FLAG_GRANT_READ_URI_PERMISSION) + } + context.startActivity(intent) + } catch (e: Exception) { + withContext(Dispatchers.Main) { + android.widget.Toast.makeText(context, "Cannot play video: ${e.message}", android.widget.Toast.LENGTH_SHORT).show() + } + } + } + } + ) { + Row( + modifier = Modifier.padding(12.dp), + verticalAlignment = Alignment.CenterVertically + ) { + Icon( + painter = painterResource(id = R.drawable.ic_stop), + contentDescription = "Play Video", + tint = MaterialTheme.colorScheme.primary, + modifier = Modifier.size(32.dp) + ) + Spacer(modifier = Modifier.width(12.dp)) + Column { + Text( + text = attachment.fileName, + style = MaterialTheme.typography.bodyMedium, + fontWeight = FontWeight.Bold, + maxLines = 1, + overflow = TextOverflow.Ellipsis + ) + Text( + text = "Tap to play video", + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.7f) + ) + } + } + } +} + +@Composable +fun DocumentAttachmentBubble(attachment: Attachment) { + val context = LocalContext.current + val coroutineScope = rememberCoroutineScope() + Card( + colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.5f)), + shape = RoundedCornerShape(12.dp), + modifier = Modifier.fillMaxWidth().padding(vertical = 4.dp).clickable { + coroutineScope.launch(Dispatchers.IO) { + try { + val fileBytes = java.io.File(attachment.filePath).readBytes() + val decrypted = com.app.assistant.db.EncryptionUtil.decryptFile(fileBytes, attachment.iv) + val ext = attachment.fileName.substringAfterLast('.', "pdf") + val temp = java.io.File(context.cacheDir, "doc_${attachment.id}.$ext") + temp.writeBytes(decrypted) + + val uri = androidx.core.content.FileProvider.getUriForFile( + context, + "com.app.assistant.fileprovider", + temp + ) + val intent = android.content.Intent(android.content.Intent.ACTION_VIEW).apply { + setDataAndType(uri, attachment.mimeType) + addFlags(android.content.Intent.FLAG_GRANT_READ_URI_PERMISSION) + } + context.startActivity(intent) + } catch (e: Exception) { + withContext(Dispatchers.Main) { + android.widget.Toast.makeText(context, "Cannot open document: ${e.message}", android.widget.Toast.LENGTH_SHORT).show() + } + } + } + } + ) { + Row( + modifier = Modifier.padding(12.dp), + verticalAlignment = Alignment.CenterVertically + ) { + Icon( + painter = painterResource(id = R.drawable.ic_translate), + contentDescription = "Open Document", + tint = MaterialTheme.colorScheme.primary, + modifier = Modifier.size(32.dp) + ) + Spacer(modifier = Modifier.width(12.dp)) + Column { + Text( + text = attachment.fileName, + style = MaterialTheme.typography.bodyMedium, + fontWeight = FontWeight.Bold, + maxLines = 1, + overflow = TextOverflow.Ellipsis + ) + val subText = if (attachment.mimeType.startsWith("text/") || attachment.mimeType == "application/json") { + "Text appended inline (Tap to open)" + } else { + "Tap to open document" + } + Text( + text = subText, + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.7f) + ) + } } } } diff --git a/app/src/main/java/com/app/assistant/ui/screen/MyLayout.kt b/app/src/main/java/com/app/assistant/ui/screen/MyLayout.kt index 17ac757..132c553 100644 --- a/app/src/main/java/com/app/assistant/ui/screen/MyLayout.kt +++ b/app/src/main/java/com/app/assistant/ui/screen/MyLayout.kt @@ -25,6 +25,23 @@ import androidx.compose.ui.tooling.preview.Preview import com.app.assistant.ui.theme.AssistantTheme import androidx.compose.ui.focus.FocusManager import androidx.compose.ui.platform.SoftwareKeyboardController +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Warning +import androidx.compose.material.icons.filled.Close +import androidx.compose.material3.Icon +import androidx.compose.material3.Text +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.IconButton +import androidx.compose.ui.graphics.Color +import androidx.compose.runtime.remember +import androidx.compose.runtime.getValue +import androidx.compose.runtime.setValue +import androidx.compose.runtime.mutableStateOf @Composable @Suppress("ktlint:standard:function-naming") @@ -43,6 +60,13 @@ fun MyLayout( onStartListening: (onResult: (String) -> Unit, onPartialResult: (String) -> Unit) -> Unit, onProcessQuestion: (FocusManager, SoftwareKeyboardController, Boolean) -> Unit, isTranslateEnabled: Boolean, + selectedAttachments: List = emptyList(), + onRemoveAttachment: (com.app.assistant.model.Attachment) -> Unit = {}, + onAttachClick: (String) -> Unit = {}, + isImageSupported: Boolean = false, + isAudioSupported: Boolean = false, + isVideoSupported: Boolean = false, + isDocumentSupported: Boolean = false, ) { val keyboardController = LocalSoftwareKeyboardController.current val focusManager = LocalFocusManager.current @@ -96,6 +120,68 @@ fun MyLayout( } } + var isDismissed by remember(chatList.size, isImageSupported, isAudioSupported, isVideoSupported, isDocumentSupported) { + mutableStateOf(false) + } + + val showWarning = remember(chatList.size, isImageSupported, isAudioSupported, isVideoSupported, isDocumentSupported) { + chatList.any { conversation -> + conversation.attachments.any { att -> + val isTextDoc = att.mimeType.startsWith("text/") || att.mimeType == "application/json" + if (isTextDoc) { + false + } else { + when { + att.mimeType.startsWith("image/") -> !isImageSupported + att.mimeType.startsWith("audio/") -> !isAudioSupported + att.mimeType.startsWith("video/") -> !isVideoSupported + att.mimeType.startsWith("application/pdf") -> !isDocumentSupported + else -> true + } + } + } + } + } + + if (showWarning && !isDismissed) { + Row( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 16.dp, vertical = 4.dp) + .background( + color = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.6f), + shape = RoundedCornerShape(12.dp) + ) + .padding(12.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp) + ) { + Icon( + imageVector = Icons.Default.Warning, + contentDescription = "Warning", + tint = MaterialTheme.colorScheme.primary, + modifier = Modifier.size(20.dp) + ) + Text( + text = "Current model does not support some media in this chat. Incompatible past attachments will be ignored by the LLM.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.weight(1f) + ) + IconButton( + onClick = { isDismissed = true }, + modifier = Modifier.size(20.dp) + ) { + Icon( + imageVector = Icons.Default.Close, + contentDescription = "Dismiss warning", + tint = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.7f), + modifier = Modifier.size(16.dp) + ) + } + } + } + if (keyboardController != null) { UserInputField( focusManager = focusManager, @@ -107,6 +193,13 @@ fun MyLayout( onStopSpeaking = onStopSpeaking, onStartListening = onStartListening, onProcessQuestion = onProcessQuestion, + selectedAttachments = selectedAttachments, + onRemoveAttachment = onRemoveAttachment, + onAttachClick = onAttachClick, + isImageSupported = isImageSupported, + isAudioSupported = isAudioSupported, + isVideoSupported = isVideoSupported, + isDocumentSupported = isDocumentSupported, ) } } diff --git a/app/src/main/java/com/app/assistant/ui/screen/SettingsScreen.kt b/app/src/main/java/com/app/assistant/ui/screen/SettingsScreen.kt index 4f6512d..aa9812a 100644 --- a/app/src/main/java/com/app/assistant/ui/screen/SettingsScreen.kt +++ b/app/src/main/java/com/app/assistant/ui/screen/SettingsScreen.kt @@ -664,6 +664,7 @@ fun VerifySection( CapabilityRow("Vision / Image Support", verificationState.capabilities.hasImageInput) CapabilityRow("Audio Support", verificationState.capabilities.hasAudioInput) CapabilityRow("Video Support", verificationState.capabilities.hasVideoInput) + CapabilityRow("Document Support (PDF/TXT)", verificationState.capabilities.hasDocumentInput) Spacer(modifier = Modifier.height(12.dp)) OutlinedButton( diff --git a/app/src/main/java/com/app/assistant/ui/screen/UserInputField.kt b/app/src/main/java/com/app/assistant/ui/screen/UserInputField.kt index babdfb0..6b07090 100644 --- a/app/src/main/java/com/app/assistant/ui/screen/UserInputField.kt +++ b/app/src/main/java/com/app/assistant/ui/screen/UserInputField.kt @@ -4,52 +4,73 @@ import androidx.compose.animation.core.Animatable import androidx.compose.animation.core.LinearEasing import androidx.compose.animation.core.tween import androidx.compose.foundation.Canvas +import androidx.compose.foundation.Image +import androidx.compose.foundation.background +import androidx.compose.ui.draw.clip import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.imePadding import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.lazy.LazyRow +import androidx.compose.foundation.lazy.items +import androidx.compose.foundation.shape.CircleShape import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.material.icons.Icons import androidx.compose.material.icons.automirrored.filled.Send +import androidx.compose.material.icons.filled.Close +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.DropdownMenu +import androidx.compose.material3.DropdownMenuItem import androidx.compose.material3.Icon import androidx.compose.material3.IconButton +import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Text import androidx.compose.material3.TextField import androidx.compose.material3.TextFieldDefaults import androidx.compose.runtime.Composable -import androidx.compose.ui.res.stringResource import androidx.compose.runtime.LaunchedEffect -import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier -import androidx.compose.ui.draw.clip -import androidx.compose.ui.focus.FocusManager import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.graphics.luminance +import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.layout.onGloballyPositioned import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.SoftwareKeyboardController import androidx.compose.ui.res.painterResource +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.unit.IntSize import androidx.compose.ui.unit.dp -import androidx.compose.ui.tooling.preview.Preview -import com.app.assistant.ui.theme.AssistantTheme +import androidx.compose.ui.unit.sp import com.app.assistant.R +import com.app.assistant.model.Attachment +import com.app.assistant.ui.theme.AssistantTheme +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.isActive import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext import kotlin.math.hypot @Composable @Suppress("ktlint:standard:function-naming") fun UserInputField( - focusManager: FocusManager, + focusManager: androidx.compose.ui.focus.FocusManager, keyboardController: SoftwareKeyboardController, isSpeaking: Boolean, isListening: Boolean, @@ -57,8 +78,15 @@ fun UserInputField( onQuestionChange: (String) -> Unit, onStopSpeaking: () -> Unit, onStartListening: (onResult: (String) -> Unit, onPartialResult: (String) -> Unit) -> Unit, - onProcessQuestion: (FocusManager, SoftwareKeyboardController, Boolean) -> Unit, + onProcessQuestion: (androidx.compose.ui.focus.FocusManager, SoftwareKeyboardController, Boolean) -> Unit, modifier: Modifier = Modifier, + selectedAttachments: List = emptyList(), + onRemoveAttachment: (Attachment) -> Unit = {}, + onAttachClick: (String) -> Unit = {}, + isImageSupported: Boolean = false, + isAudioSupported: Boolean = false, + isVideoSupported: Boolean = false, + isDocumentSupported: Boolean = false, ) { val containerColor = TextFieldDefaults.colors().unfocusedContainerColor val rippleColor = adjustContrast(containerColor) @@ -99,99 +127,274 @@ fun UserInputField( } } - Box( - modifier = - modifier + Column( + modifier = modifier + .fillMaxWidth() + .imePadding() + .padding(8.dp) + ) { + if (selectedAttachments.isNotEmpty()) { + AttachmentPreviewList( + attachments = selectedAttachments, + onRemove = onRemoveAttachment, + modifier = Modifier.padding(bottom = 8.dp) + ) + } + + Box( + modifier = Modifier .fillMaxWidth() - .imePadding() - .padding(8.dp) .clip(RoundedCornerShape(20)) // clip ripple inside .onGloballyPositioned { coords -> textFieldSize = coords.size }, - ) { - TextField( - modifier = - Modifier - .fillMaxWidth(), - shape = RoundedCornerShape(20), - value = question, - placeholder = { Text(stringResource(id = R.string.type_here_placeholder)) }, - maxLines = 2, - onValueChange = onQuestionChange, - colors = - TextFieldDefaults.colors( + ) { + TextField( + modifier = Modifier.fillMaxWidth(), + shape = RoundedCornerShape(20), + value = question, + placeholder = { Text(stringResource(id = R.string.type_here_placeholder)) }, + maxLines = 4, + onValueChange = onQuestionChange, + colors = TextFieldDefaults.colors( focusedIndicatorColor = Color.Transparent, unfocusedIndicatorColor = Color.Transparent, disabledIndicatorColor = Color.Transparent, ), - trailingIcon = { - Row(horizontalArrangement = Arrangement.End) { - if (isSpeaking) { + leadingIcon = { + val showAttachmentButton = isImageSupported || isAudioSupported || isVideoSupported || isDocumentSupported + if (showAttachmentButton && !isSpeaking) { + var menuExpanded by remember { mutableStateOf(false) } + Box { + IconButton(onClick = { menuExpanded = true }) { + Icon( + painter = painterResource(id = R.drawable.ic_attachment), + contentDescription = "Attach file", + tint = MaterialTheme.colorScheme.onSurfaceVariant + ) + } + DropdownMenu( + expanded = menuExpanded, + onDismissRequest = { menuExpanded = false } + ) { + if (isImageSupported) { + DropdownMenuItem( + text = { Text("Image") }, + onClick = { + menuExpanded = false + onAttachClick("image/*") + } + ) + DropdownMenuItem( + text = { Text("Take Photo (Camera)") }, + onClick = { + menuExpanded = false + onAttachClick("camera") + } + ) + } + if (isAudioSupported) { + DropdownMenuItem( + text = { Text("Audio") }, + onClick = { + menuExpanded = false + onAttachClick("audio/*") + } + ) + } + if (isVideoSupported) { + DropdownMenuItem( + text = { Text("Video") }, + onClick = { + menuExpanded = false + onAttachClick("video/*") + } + ) + } + if (isDocumentSupported) { + DropdownMenuItem( + text = { Text("Document (PDF)") }, + onClick = { + menuExpanded = false + onAttachClick("application/pdf") + } + ) + } + // Always support text files as they are decrypted and appended inline as text + DropdownMenuItem( + text = { Text("Text Document (.txt/.csv/.json)") }, + onClick = { + menuExpanded = false + onAttachClick("text/*") + } + ) + } + } + } + }, + trailingIcon = { + Row( + horizontalArrangement = Arrangement.End, + verticalAlignment = Alignment.CenterVertically + ) { + if (isSpeaking) { + IconButton(onClick = { + if (isSpeaking) { + onStopSpeaking() + } + }) { + Icon( + painter = painterResource(id = R.drawable.ic_stop), + contentDescription = stringResource(id = R.string.stop_desc), + ) + } + } else { + IconButton(onClick = { + onStartListening( + { recognizedText -> + onQuestionChange(recognizedText) + if (recognizedText.isNotEmpty()) { + onProcessQuestion(focusManager, keyboardController, true) + } + }, + { recognizedText -> + onQuestionChange(recognizedText) + }, + ) + }) { + Icon( + painter = painterResource(id = R.drawable.ic_mic), + contentDescription = stringResource(id = R.string.mic_desc), + ) + } + } IconButton(onClick = { - if (isSpeaking) { - onStopSpeaking() + if (question.isNotEmpty() || selectedAttachments.isNotEmpty()) { + onProcessQuestion(focusManager, keyboardController, false) } }) { Icon( - painter = painterResource(id = R.drawable.ic_stop), - contentDescription = stringResource(id = R.string.stop_desc), - ) - } - } else { - IconButton(onClick = { - onStartListening( - { recognizedText -> - onQuestionChange(recognizedText) - if (recognizedText.isNotEmpty()) { - onProcessQuestion( - focusManager, - keyboardController, - true, - ) - } - }, - { recognizedText -> - onQuestionChange(recognizedText) - }, - ) - }) { - Icon( - painter = painterResource(id = R.drawable.ic_mic), - contentDescription = stringResource(id = R.string.mic_desc), + imageVector = Icons.AutoMirrored.Filled.Send, + contentDescription = stringResource(id = R.string.send_desc), ) } } - IconButton(onClick = { - if (question.isNotEmpty()) { - onProcessQuestion( - focusManager, - keyboardController, - false, - ) - } - }) { - Icon( - imageVector = Icons.AutoMirrored.Filled.Send, - contentDescription = stringResource(id = R.string.send_desc), - ) - } - } - }, - ) + }, + ) - Canvas( - modifier = - Modifier + Canvas( + modifier = Modifier .matchParentSize() .clip(RoundedCornerShape(20)), - ) { - if (rippleAlpha.value > 0f) { - drawCircle( - color = rippleColor.copy(alpha = rippleAlpha.value), - radius = rippleRadius.value, - center = center, + ) { + if (rippleAlpha.value > 0f) { + drawCircle( + color = rippleColor.copy(alpha = rippleAlpha.value), + radius = rippleRadius.value, + center = center, + ) + } + } + } + } +} + +@Composable +fun AttachmentPreviewList( + attachments: List, + onRemove: (Attachment) -> Unit, + modifier: Modifier = Modifier +) { + LazyRow( + modifier = modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.spacedBy(8.dp) + ) { + items(attachments) { attachment -> + AttachmentPreviewCard(attachment = attachment, onRemove = { onRemove(attachment) }) + } + } +} + +@Composable +fun AttachmentPreviewCard( + attachment: Attachment, + onRemove: () -> Unit +) { + Box( + modifier = Modifier + .size(72.dp) + .clip(RoundedCornerShape(8.dp)) + .background(MaterialTheme.colorScheme.surfaceVariant) + ) { + if (attachment.mimeType.startsWith("image/")) { + var bitmap by remember { mutableStateOf(null) } + LaunchedEffect(attachment) { + withContext(Dispatchers.IO) { + try { + val fileBytes = java.io.File(attachment.filePath).readBytes() + val decrypted = com.app.assistant.db.EncryptionUtil.decryptFile(fileBytes, attachment.iv) + bitmap = android.graphics.BitmapFactory.decodeByteArray(decrypted, 0, decrypted.size) + } catch (e: Exception) { + e.printStackTrace() + } + } + } + if (bitmap != null) { + Image( + bitmap = bitmap!!.asImageBitmap(), + contentDescription = null, + modifier = Modifier.fillMaxSize(), + contentScale = ContentScale.Crop + ) + } else { + Box(modifier = Modifier.fillMaxSize(), contentAlignment = Alignment.Center) { + CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp) + } + } + } else { + Column( + modifier = Modifier + .fillMaxSize() + .padding(4.dp), + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.Center + ) { + val iconRes = when { + attachment.mimeType.startsWith("audio/") -> R.drawable.ic_mic + attachment.mimeType.startsWith("video/") -> R.drawable.ic_stop + else -> R.drawable.ic_translate + } + Icon( + painter = painterResource(id = iconRes), + contentDescription = null, + modifier = Modifier.size(24.dp), + tint = MaterialTheme.colorScheme.primary + ) + Spacer(modifier = Modifier.height(2.dp)) + Text( + text = attachment.fileName, + fontSize = 8.sp, + maxLines = 1, + overflow = TextOverflow.Ellipsis, + textAlign = TextAlign.Center, + color = MaterialTheme.colorScheme.onSurfaceVariant ) } } + + IconButton( + onClick = onRemove, + modifier = Modifier + .align(Alignment.TopEnd) + .size(18.dp) + .background(Color.Black.copy(alpha = 0.6f), shape = CircleShape) + .padding(2.dp) + ) { + Icon( + imageVector = Icons.Default.Close, + contentDescription = "Remove", + tint = Color.White, + modifier = Modifier.size(12.dp) + ) + } } } @@ -201,7 +404,6 @@ fun adjustContrast( ): Color { val isDark = color.luminance() < 0.5 return if (isDark) { - // lighten in dark mode Color( red = (color.red + factor).coerceAtMost(1f), green = (color.green + factor).coerceAtMost(1f), @@ -209,7 +411,6 @@ fun adjustContrast( alpha = color.alpha, ) } else { - // darken in light mode Color( red = (color.red - factor).coerceAtLeast(0f), green = (color.green - factor).coerceAtLeast(0f), @@ -235,51 +436,8 @@ fun UserInputFieldIdlePreview() { onQuestionChange = {}, onStopSpeaking = {}, onStartListening = { _, _ -> }, - onProcessQuestion = { _, _, _ -> } - ) - } - } -} - -@Preview(showBackground = true) -@Composable -fun UserInputFieldSpeakingPreview() { - val focusManager = androidx.compose.ui.platform.LocalFocusManager.current - val keyboardController = androidx.compose.ui.platform.LocalSoftwareKeyboardController.current - AssistantTheme { - if (keyboardController != null) { - UserInputField( - focusManager = focusManager, - keyboardController = keyboardController, - isSpeaking = true, - isListening = false, - question = "This is a spoken response from the AI", - onQuestionChange = {}, - onStopSpeaking = {}, - onStartListening = { _, _ -> }, - onProcessQuestion = { _, _, _ -> } - ) - } - } -} - -@Preview(showBackground = true) -@Composable -fun UserInputFieldListeningPreview() { - val focusManager = androidx.compose.ui.platform.LocalFocusManager.current - val keyboardController = androidx.compose.ui.platform.LocalSoftwareKeyboardController.current - AssistantTheme { - if (keyboardController != null) { - UserInputField( - focusManager = focusManager, - keyboardController = keyboardController, - isSpeaking = false, - isListening = true, - question = "Listening for your voice...", - onQuestionChange = {}, - onStopSpeaking = {}, - onStartListening = { _, _ -> }, - onProcessQuestion = { _, _, _ -> } + onProcessQuestion = { _, _, _ -> }, + isImageSupported = true ) } } diff --git a/app/src/main/java/com/app/assistant/usecase/GetAiResponseUseCase.kt b/app/src/main/java/com/app/assistant/usecase/GetAiResponseUseCase.kt index 13d6e3e..8ac2d75 100644 --- a/app/src/main/java/com/app/assistant/usecase/GetAiResponseUseCase.kt +++ b/app/src/main/java/com/app/assistant/usecase/GetAiResponseUseCase.kt @@ -51,7 +51,8 @@ class GetAiResponseUseCase( client = client, apiKey = apiKey ?: "", model = model, - config = config + config = config, + provider = provider ) val systemContext = messages.find { it.role == "system" }?.content ?: "" @@ -92,7 +93,8 @@ class GetAiResponseUseCase( client = client, apiKey = apiKey ?: "", model = model, - config = config + config = config, + provider = provider ) val systemContext = messages.find { it.role == "system" }?.content ?: "" diff --git a/app/src/main/java/com/app/assistant/usecase/ProcessChatCommandUseCase.kt b/app/src/main/java/com/app/assistant/usecase/ProcessChatCommandUseCase.kt index 178f26b..1a49877 100644 --- a/app/src/main/java/com/app/assistant/usecase/ProcessChatCommandUseCase.kt +++ b/app/src/main/java/com/app/assistant/usecase/ProcessChatCommandUseCase.kt @@ -2,12 +2,19 @@ package com.app.assistant.usecase import com.app.assistant.llm.LlmMessage import com.app.assistant.model.Conversation +import com.app.assistant.repository.SettingsRepository import com.app.assistant.util.Category import com.google.mediapipe.tasks.text.textclassifier.TextClassifierResult +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.flow.collect import java.util.Locale class ProcessChatCommandUseCase( - private val getAiResponseUseCase: GetAiResponseUseCase + private val getAiResponseUseCase: GetAiResponseUseCase, + private val settingsRepository: SettingsRepository ) { fun cleanAndPunctuate(input: String): String { val trimmedInput = input.trim() @@ -60,33 +67,68 @@ class ProcessChatCommandUseCase( suspend fun getAiChatResponse( systemContext: String, chatHistory: List - ): String? { - val messages = mutableListOf() - messages.add(LlmMessage(role = "system", content = systemContext)) - - for (item in chatHistory) { - if (item.isLoading) continue - val role = if (item.isMe) "user" else "assistant" - messages.add(LlmMessage(role = role, content = item.text)) - } - - return getAiResponseUseCase.execute(messages) + ): String? = withContext(Dispatchers.IO) { + val messages = mapConversationsToLlmMessages(systemContext, chatHistory) + getAiResponseUseCase.execute(messages) } fun getAiChatResponseStream( systemContext: String, chatHistory: List - ): kotlinx.coroutines.flow.Flow { + ): kotlinx.coroutines.flow.Flow = flow { + val messages = withContext(Dispatchers.IO) { + mapConversationsToLlmMessages(systemContext, chatHistory) + } + getAiResponseUseCase.executeStream(messages).collect { + emit(it) + } + }.flowOn(Dispatchers.IO) + + private fun mapConversationsToLlmMessages( + systemContext: String, + chatHistory: List + ): List { val messages = mutableListOf() messages.add(LlmMessage(role = "system", content = systemContext)) for (item in chatHistory) { if (item.isLoading) continue val role = if (item.isMe) "user" else "assistant" - messages.add(LlmMessage(role = role, content = item.text)) + val llmAttachments = if (item.attachments.isNotEmpty()) { + val filtered = item.attachments.filter { att -> + if (att.mimeType.startsWith("text/") || att.mimeType == "application/json") { + false + } else { + when { + att.mimeType.startsWith("image/") -> settingsRepository.getIsImageSupported() + att.mimeType.startsWith("audio/") -> settingsRepository.getIsAudioSupported() + att.mimeType.startsWith("video/") -> settingsRepository.getIsVideoSupported() + att.mimeType.startsWith("application/pdf") -> settingsRepository.getIsDocumentSupported() + else -> false + } + } + } + if (filtered.isNotEmpty()) { + filtered.map { att -> + val file = java.io.File(att.filePath) + val base64 = if (file.exists()) { + val encryptedBytes = file.readBytes() + val decryptedBytes = com.app.assistant.db.EncryptionUtil.decryptFile(encryptedBytes, att.iv) + android.util.Base64.encodeToString(decryptedBytes, android.util.Base64.NO_WRAP) + } else { + "" + } + com.app.assistant.llm.LlmAttachment( + base64Data = base64, + mimeType = att.mimeType, + fileName = att.fileName + ) + } + } else null + } else null + messages.add(LlmMessage(role = role, content = item.text, attachments = llmAttachments)) } - - return getAiResponseUseCase.executeStream(messages) + return messages } companion object { diff --git a/app/src/main/java/com/app/assistant/viewmodel/MainViewModel.kt b/app/src/main/java/com/app/assistant/viewmodel/MainViewModel.kt index b29c9c0..a626c8b 100644 --- a/app/src/main/java/com/app/assistant/viewmodel/MainViewModel.kt +++ b/app/src/main/java/com/app/assistant/viewmodel/MainViewModel.kt @@ -11,6 +11,7 @@ import com.app.assistant.db.DynamicConversationRepository import com.app.assistant.db.SyncStateList import com.app.assistant.model.Conversation import com.app.assistant.model.Group +import com.app.assistant.model.Attachment import com.app.assistant.repository.SettingsRepository import com.app.assistant.translation.TranslatorManager import com.app.assistant.usecase.CallContactUseCase @@ -36,6 +37,7 @@ import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.collect import kotlinx.coroutines.launch import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import java.lang.reflect.Modifier import java.net.URI import kotlin.coroutines.resume @@ -57,6 +59,190 @@ class MainViewModel( private val _question = MutableStateFlow("") val question: StateFlow = _question.asStateFlow() + private val _selectedAttachments = MutableStateFlow>(emptyList()) + val selectedAttachments: StateFlow> = _selectedAttachments.asStateFlow() + + private fun compressImageIfNeeded(context: android.content.Context, uri: android.net.Uri): ByteArray? { + val mimeType = context.contentResolver.getType(uri) ?: return null + if (!mimeType.startsWith("image/")) { + return null + } + try { + // First read EXIF rotation + var rotation = 0 + try { + context.contentResolver.openInputStream(uri)?.use { stream -> + val exif = android.media.ExifInterface(stream) + val orientation = exif.getAttributeInt( + android.media.ExifInterface.TAG_ORIENTATION, + android.media.ExifInterface.ORIENTATION_NORMAL + ) + rotation = when (orientation) { + android.media.ExifInterface.ORIENTATION_ROTATE_90 -> 90 + android.media.ExifInterface.ORIENTATION_ROTATE_180 -> 180 + android.media.ExifInterface.ORIENTATION_ROTATE_270 -> 270 + else -> 0 + } + } + } catch (e: Exception) { + Log.e("MainViewModel", "Failed to parse EXIF orientation", e) + } + + // Decode dimensions only + val options = android.graphics.BitmapFactory.Options().apply { + inJustDecodeBounds = true + } + context.contentResolver.openInputStream(uri)?.use { stream -> + android.graphics.BitmapFactory.decodeStream(stream, null, options) + } + + val width = options.outWidth + val height = options.outHeight + if (width <= 0 || height <= 0) return null + + // Calculate sample size (approximate scaling down during decode) + val maxDim = 1200 + var inSampleSize = 1 + if (width > maxDim || height > maxDim) { + val halfWidth = width / 2 + val halfHeight = height / 2 + while ((halfWidth / inSampleSize) >= maxDim || (halfHeight / inSampleSize) >= maxDim) { + inSampleSize *= 2 + } + } + + // Decode full image with sample size + val decodeOptions = android.graphics.BitmapFactory.Options().apply { + this.inSampleSize = inSampleSize + } + val bitmap = context.contentResolver.openInputStream(uri)?.use { stream -> + android.graphics.BitmapFactory.decodeStream(stream, null, decodeOptions) + } ?: return null + + // Precise scaling to max 1200px + val currentWidth = bitmap.width + val currentHeight = bitmap.height + val scaledBitmap = if (currentWidth > maxDim || currentHeight > maxDim) { + val ratio = currentWidth.toFloat() / currentHeight.toFloat() + val (newWidth, newHeight) = if (currentWidth > currentHeight) { + maxDim to (maxDim / ratio).toInt() + } else { + (maxDim * ratio).toInt() to maxDim + } + android.graphics.Bitmap.createScaledBitmap(bitmap, newWidth, newHeight, true).also { + if (it != bitmap) { + bitmap.recycle() + } + } + } else { + bitmap + } + + // Correct EXIF rotation + val rotatedBitmap = if (rotation != 0) { + val matrix = android.graphics.Matrix().apply { postRotate(rotation.toFloat()) } + android.graphics.Bitmap.createBitmap( + scaledBitmap, 0, 0, scaledBitmap.width, scaledBitmap.height, matrix, true + ).also { + if (it != scaledBitmap) { + scaledBitmap.recycle() + } + } + } else { + scaledBitmap + } + + // Compress to JPEG 80% + val outputStream = java.io.ByteArrayOutputStream() + rotatedBitmap.compress(android.graphics.Bitmap.CompressFormat.JPEG, 80, outputStream) + val compressedBytes = outputStream.toByteArray() + rotatedBitmap.recycle() + return compressedBytes + } catch (e: Exception) { + Log.e("MainViewModel", "Error compressing image", e) + return null + } + } + + fun addSelectedAttachment(uri: android.net.Uri) { + viewModelScope.launch(Dispatchers.IO) { + val context = getApplication().applicationContext + val contentResolver = context.contentResolver + var fileName = "file_${System.currentTimeMillis()}" + var mimeType = contentResolver.getType(uri) ?: "*/*" + contentResolver.query(uri, null, null, null, null)?.use { cursor -> + val nameIndex = cursor.getColumnIndex(android.provider.OpenableColumns.DISPLAY_NAME) + if (nameIndex != -1 && cursor.moveToFirst()) { + fileName = cursor.getString(nameIndex) + } + } + try { + val isImage = mimeType.startsWith("image/") || + fileName.endsWith(".jpg", ignoreCase = true) || + fileName.endsWith(".jpeg", ignoreCase = true) || + fileName.endsWith(".png", ignoreCase = true) || + fileName.endsWith(".webp", ignoreCase = true) + + var fileBytes: ByteArray? = null + if (isImage) { + fileBytes = compressImageIfNeeded(context, uri) + if (fileBytes != null) { + mimeType = "image/jpeg" + if (!fileName.lowercase().endsWith(".jpg") && !fileName.lowercase().endsWith(".jpeg")) { + val dotIndex = fileName.lastIndexOf('.') + fileName = if (dotIndex != -1) { + "${fileName.substring(0, dotIndex)}.jpg" + } else { + "$fileName.jpg" + } + } + } + } + + if (fileBytes == null) { + val inputStream = contentResolver.openInputStream(uri) ?: return@launch + fileBytes = inputStream.readBytes() + inputStream.close() + } + + val iv = com.app.assistant.db.EncryptionUtil.generateIV() + val encryptedBytes = com.app.assistant.db.EncryptionUtil.encryptFile(fileBytes, iv) + + val attachmentsDir = java.io.File(context.filesDir, "attachments") + if (!attachmentsDir.exists()) { + attachmentsDir.mkdirs() + } + + val targetFile = java.io.File(attachmentsDir, "file_${System.currentTimeMillis()}_${java.util.UUID.randomUUID()}.enc") + targetFile.writeBytes(encryptedBytes) + + val attachment = Attachment( + filePath = targetFile.absolutePath, + mimeType = mimeType, + fileName = fileName, + iv = iv + ) + _selectedAttachments.value = _selectedAttachments.value + attachment + } catch (e: Exception) { + Log.e("MainViewModel", "Failed to add attachment", e) + } + } + } + + fun removeSelectedAttachment(attachment: Attachment) { + viewModelScope.launch(Dispatchers.IO) { + try { + val file = java.io.File(attachment.filePath) + if (file.exists()) { + file.delete() + } + } catch (e: Exception) { + e.printStackTrace() + } + _selectedAttachments.value = _selectedAttachments.value - attachment + } + } + fun setQuestion(text: String) { _question.value = text } @@ -268,13 +454,37 @@ class MainViewModel( speak: Boolean = false, ) { val originalQuestion = question.value + val currentAttachments = _selectedAttachments.value + _selectedAttachments.value = emptyList() + focusManager?.clearFocus() keyboardController?.hide() setQuestion("") viewModelScope.launch { val itemId: Long - val newItem = Conversation(text = originalQuestion, isMe = true) + val processedQuestion = processChatCommandUseCase.cleanAndPunctuate(originalQuestion) + val finalQuestionText = withContext(Dispatchers.IO) { + val finalQuestionBuilder = java.lang.StringBuilder(processedQuestion) + currentAttachments.forEach { att -> + if (att.mimeType.startsWith("text/") || att.mimeType == "application/json") { + try { + val file = java.io.File(att.filePath) + if (file.exists()) { + val encryptedBytes = file.readBytes() + val decryptedBytes = com.app.assistant.db.EncryptionUtil.decryptFile(encryptedBytes, att.iv) + val textContent = String(decryptedBytes, Charsets.UTF_8) + finalQuestionBuilder.append("\n\n[Attached File: ${att.fileName}]\n---\n$textContent\n---\n") + } + } catch (e: java.lang.Exception) { + Log.e("MainViewModel", "Failed to extract text file content", e) + } + } + } + finalQuestionBuilder.toString() + } + + val newItem = Conversation(text = finalQuestionText, isMe = true, attachments = currentAttachments) chatList.add(newItem) if (_isCustomUIHalfPage.value) { @@ -287,12 +497,6 @@ class MainViewModel( chatList.add(loadingItem) val loadingItemId = loadingItem.id - val processedQuestion = processChatCommandUseCase.cleanAndPunctuate(originalQuestion) - chatList.indexOfFirst { it.id == itemId }.takeIf { it != -1 }?.let { index -> - val updatedItem = chatList[index].copy(text = processedQuestion) - chatList.set(index, updatedItem) - } - if (lockState != LockState.None && processChatCommandUseCase.isNegativeOrNotRequired(processedQuestion)) { lockState = LockState.None classifierHelper.classify(processedQuestion, itemId, loadingItemId, speak) diff --git a/app/src/main/java/com/app/assistant/viewmodel/MainViewModelFactory.kt b/app/src/main/java/com/app/assistant/viewmodel/MainViewModelFactory.kt index ac9f4b5..a5f090f 100644 --- a/app/src/main/java/com/app/assistant/viewmodel/MainViewModelFactory.kt +++ b/app/src/main/java/com/app/assistant/viewmodel/MainViewModelFactory.kt @@ -42,7 +42,7 @@ class MainViewModelFactory( val getWeatherUseCase = GetWeatherUseCase(permissionChecker, weatherRepo, getAiResponseUseCase) val setAlarmUseCase = SetAlarmUseCase(resourceProvider) val setReminderUseCase = SetReminderUseCase(resourceProvider) - val processChatCommandUseCase = ProcessChatCommandUseCase(getAiResponseUseCase) + val processChatCommandUseCase = ProcessChatCommandUseCase(getAiResponseUseCase, settingsRepo) @Suppress("UNCHECKED_CAST") return MainViewModel( diff --git a/app/src/main/java/com/app/assistant/viewmodel/SettingsViewModel.kt b/app/src/main/java/com/app/assistant/viewmodel/SettingsViewModel.kt index 99557b8..987ef0a 100644 --- a/app/src/main/java/com/app/assistant/viewmodel/SettingsViewModel.kt +++ b/app/src/main/java/com/app/assistant/viewmodel/SettingsViewModel.kt @@ -154,19 +154,22 @@ class SettingsViewModel( ) // 3. Audio/Video checks (currently Gemini 1.5/2.0 is the main provider that supports audio/video input natively in the API) - val hasAudio = provider == LlmProvider.GEMINI && (model.contains("1.5") || model.contains("2.0")) - val hasVideo = provider == LlmProvider.GEMINI && (model.contains("1.5") || model.contains("2.0")) + val hasAudio = provider == LlmProvider.GEMINI && (model.contains("1.5") || model.contains("2.0") || model.contains("2.5")) + val hasVideo = provider == LlmProvider.GEMINI && (model.contains("1.5") || model.contains("2.0") || model.contains("2.5")) + val hasDoc = (provider == LlmProvider.GEMINI && (model.contains("1.5") || model.contains("2.0") || model.contains("2.5"))) || (provider == LlmProvider.ANTHROPIC && model.contains("claude-3")) val capabilities = ModelCapabilities( hasImageInput = isVisionOk, hasAudioInput = hasAudio, - hasVideoInput = hasVideo + hasVideoInput = hasVideo, + hasDocumentInput = hasDoc ) // Save detected capabilities in repository settingsRepository.setIsImageSupported(isVisionOk) settingsRepository.setIsAudioSupported(hasAudio) settingsRepository.setIsVideoSupported(hasVideo) + settingsRepository.setIsDocumentSupported(hasDoc) settingsRepository.setIsModelVerified(true) _verificationState.value = VerificationState.Success(capabilities) diff --git a/app/src/main/res/drawable/ic_attachment.xml b/app/src/main/res/drawable/ic_attachment.xml new file mode 100644 index 0000000..fe04f24 --- /dev/null +++ b/app/src/main/res/drawable/ic_attachment.xml @@ -0,0 +1,10 @@ + + + diff --git a/app/src/main/res/xml/file_paths.xml b/app/src/main/res/xml/file_paths.xml new file mode 100644 index 0000000..e5e18b5 --- /dev/null +++ b/app/src/main/res/xml/file_paths.xml @@ -0,0 +1,4 @@ + + + +