- Use camerax to implement image capture in-app instead of using the native camera app to reduce the possibility of app being killed due to GPU OOM.

- Show accelerator name in chat message sender labels.
- Attach download workers with silent foreground notifications to make them less likely to be killed.
- Update app icon to be consistent with Google style.
- Bump up version to 1.0.2.
This commit is contained in:
Jing Jin 2025-05-21 17:34:35 -07:00
parent f50d065575
commit 517f220d1b
36 changed files with 284 additions and 69 deletions

View file

@ -31,7 +31,7 @@ android {
minSdk = 26 minSdk = 26
targetSdk = 35 targetSdk = 35
versionCode = 1 versionCode = 1
versionName = "1.0.1" versionName = "1.0.2"
// Needed for HuggingFace auth workflows. // Needed for HuggingFace auth workflows.
manifestPlaceholders["appAuthRedirectScheme"] = "com.google.ai.edge.gallery.oauth" manifestPlaceholders["appAuthRedirectScheme"] = "com.google.ai.edge.gallery.oauth"

View file

@ -18,8 +18,8 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android" <manifest xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools"> xmlns:tools="http://schemas.android.com/tools">
<!-- <uses-permission android:name="android.permission.FOREGROUND_SERVICE"/>--> <uses-permission android:name="android.permission.FOREGROUND_SERVICE"/>
<!-- <uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC"/>--> <uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC"/>
<uses-permission android:name="android.permission.INTERNET" /> <uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" /> <uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
<uses-permission android:name="android.permission.CAMERA" /> <uses-permission android:name="android.permission.CAMERA" />
@ -83,11 +83,12 @@
android:resource="@xml/file_paths" /> android:resource="@xml/file_paths" />
</provider> </provider>
<!-- <service--> <service
<!-- android:name=".GalleryService"--> android:name="androidx.work.impl.foreground.SystemForegroundService"
<!-- android:foregroundServiceType="dataSync"--> android:foregroundServiceType="dataSync"
<!-- android:exported="false">--> android:exported="false"
<!-- </service>--> tools:node="merge">
</service>
</application> </application>
</manifest> </manifest>

View file

@ -18,6 +18,7 @@ package com.google.ai.edge.gallery.data
// Keys used to send/receive data to Work. // Keys used to send/receive data to Work.
const val KEY_MODEL_URL = "KEY_MODEL_URL" const val KEY_MODEL_URL = "KEY_MODEL_URL"
const val KEY_MODEL_NAME = "KEY_MODEL_NAME"
const val KEY_MODEL_VERSION = "KEY_MODEL_VERSION" const val KEY_MODEL_VERSION = "KEY_MODEL_VERSION"
const val KEY_MODEL_DOWNLOAD_MODEL_DIR = "KEY_MODEL_DOWNLOAD_MODEL_DIR" const val KEY_MODEL_DOWNLOAD_MODEL_DIR = "KEY_MODEL_DOWNLOAD_MODEL_DIR"
const val KEY_MODEL_DOWNLOAD_FILE_NAME = "KEY_MODEL_DOWNLOAD_FILE_NAME" const val KEY_MODEL_DOWNLOAD_FILE_NAME = "KEY_MODEL_DOWNLOAD_FILE_NAME"

View file

@ -24,7 +24,6 @@ import android.content.Context
import android.content.Intent import android.content.Intent
import android.content.pm.PackageManager import android.content.pm.PackageManager
import android.net.Uri import android.net.Uri
import android.os.Build
import android.util.Log import android.util.Log
import androidx.core.app.ActivityCompat import androidx.core.app.ActivityCompat
import androidx.core.app.NotificationCompat import androidx.core.app.NotificationCompat
@ -92,7 +91,8 @@ class DefaultDownloadRepository(
val builder = Data.Builder() val builder = Data.Builder()
val totalBytes = model.totalBytes + model.extraDataFiles.sumOf { it.sizeInBytes } val totalBytes = model.totalBytes + model.extraDataFiles.sumOf { it.sizeInBytes }
val inputDataBuilder = val inputDataBuilder =
builder.putString(KEY_MODEL_URL, model.url).putString(KEY_MODEL_VERSION, model.version) builder.putString(KEY_MODEL_NAME, model.name).putString(KEY_MODEL_URL, model.url)
.putString(KEY_MODEL_VERSION, model.version)
.putString(KEY_MODEL_DOWNLOAD_MODEL_DIR, model.normalizedName) .putString(KEY_MODEL_DOWNLOAD_MODEL_DIR, model.normalizedName)
.putString(KEY_MODEL_DOWNLOAD_FILE_NAME, model.downloadFileName) .putString(KEY_MODEL_DOWNLOAD_FILE_NAME, model.downloadFileName)
.putBoolean(KEY_MODEL_IS_ZIP, model.isZip).putString(KEY_MODEL_UNZIPPED_DIR, model.unzipDir) .putBoolean(KEY_MODEL_IS_ZIP, model.isZip).putString(KEY_MODEL_UNZIPPED_DIR, model.unzipDir)
@ -271,13 +271,11 @@ class DefaultDownloadRepository(
// Create the NotificationChannel, but only on API 26+ because // Create the NotificationChannel, but only on API 26+ because
// the NotificationChannel class is new and not in the support library // the NotificationChannel class is new and not in the support library
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { val importance = NotificationManager.IMPORTANCE_HIGH
val importance = NotificationManager.IMPORTANCE_HIGH val channel = NotificationChannel(channelId, channelName, importance)
val channel = NotificationChannel(channelId, channelName, importance) val notificationManager: NotificationManager =
val notificationManager: NotificationManager = context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager notificationManager.createNotificationChannel(channel)
notificationManager.createNotificationChannel(channel)
}
// Create an Intent to open your app with a deep link. // Create an Intent to open your app with a deep link.
val intent = Intent( val intent = Intent(

View file

@ -44,7 +44,10 @@ data class Classification(val label: String, val score: Float, val color: Color)
/** Base class for a chat message. */ /** Base class for a chat message. */
open class ChatMessage( open class ChatMessage(
open val type: ChatMessageType, open val side: ChatSide, open val latencyMs: Float = -1f open val type: ChatMessageType,
open val side: ChatSide,
open val latencyMs: Float = -1f,
open val accelerator: String = "",
) { ) {
open fun clone(): ChatMessage { open fun clone(): ChatMessage {
return ChatMessage(type = type, side = side, latencyMs = latencyMs) return ChatMessage(type = type, side = side, latencyMs = latencyMs)
@ -52,7 +55,8 @@ open class ChatMessage(
} }
/** Chat message for showing loading status. */ /** Chat message for showing loading status. */
class ChatMessageLoading : ChatMessage(type = ChatMessageType.LOADING, side = ChatSide.AGENT) class ChatMessageLoading(override val accelerator: String = "") :
ChatMessage(type = ChatMessageType.LOADING, side = ChatSide.AGENT, accelerator = accelerator)
/** Chat message for info (help). */ /** Chat message for info (help). */
class ChatMessageInfo(val content: String) : class ChatMessageInfo(val content: String) :
@ -79,12 +83,19 @@ open class ChatMessageText(
// Benchmark result for LLM response. // Benchmark result for LLM response.
var llmBenchmarkResult: ChatMessageBenchmarkLlmResult? = null, var llmBenchmarkResult: ChatMessageBenchmarkLlmResult? = null,
) : ChatMessage(type = ChatMessageType.TEXT, side = side, latencyMs = latencyMs) { override val accelerator: String = "",
) : ChatMessage(
type = ChatMessageType.TEXT,
side = side,
latencyMs = latencyMs,
accelerator = accelerator
) {
override fun clone(): ChatMessageText { override fun clone(): ChatMessageText {
return ChatMessageText( return ChatMessageText(
content = content, content = content,
side = side, side = side,
latencyMs = latencyMs, latencyMs = latencyMs,
accelerator = accelerator,
isMarkdown = isMarkdown, isMarkdown = isMarkdown,
llmBenchmarkResult = llmBenchmarkResult, llmBenchmarkResult = llmBenchmarkResult,
) )
@ -168,10 +179,12 @@ class ChatMessageBenchmarkLlmResult(
val statValues: MutableMap<String, Float>, val statValues: MutableMap<String, Float>,
val running: Boolean, val running: Boolean,
override val latencyMs: Float = 0f, override val latencyMs: Float = 0f,
override val accelerator: String = "",
) : ChatMessage( ) : ChatMessage(
type = ChatMessageType.BENCHMARK_LLM_RESULT, type = ChatMessageType.BENCHMARK_LLM_RESULT,
side = ChatSide.AGENT, side = ChatSide.AGENT,
latencyMs = latencyMs latencyMs = latencyMs,
accelerator = accelerator,
) )
data class Histogram( data class Histogram(

View file

@ -80,7 +80,6 @@ import androidx.compose.ui.text.AnnotatedString
import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.tooling.preview.Preview
import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.dp
import com.google.ai.edge.gallery.R import com.google.ai.edge.gallery.R
import com.google.ai.edge.gallery.data.ConfigKey
import com.google.ai.edge.gallery.data.Model import com.google.ai.edge.gallery.data.Model
import com.google.ai.edge.gallery.data.Task import com.google.ai.edge.gallery.data.Task
import com.google.ai.edge.gallery.data.TaskType import com.google.ai.edge.gallery.data.TaskType
@ -266,9 +265,13 @@ fun ChatPanel(
horizontalAlignment = hAlign, horizontalAlignment = hAlign,
) messageColumn@{ ) messageColumn@{
// Sender row. // Sender row.
var agentName = stringResource(task.agentNameRes)
if (message.accelerator.isNotEmpty()) {
agentName = "$agentName on ${message.accelerator}"
}
MessageSender( MessageSender(
message = message, message = message,
agentName = stringResource(task.agentNameRes), agentName = agentName,
imageHistoryCurIndex = imageHistoryCurIndex.intValue imageHistoryCurIndex = imageHistoryCurIndex.intValue
) )

View file

@ -141,6 +141,7 @@ open class ChatViewModel(val task: Task) : ViewModel() {
content = newContent, content = newContent,
side = lastMessage.side, side = lastMessage.side,
latencyMs = latencyMs, latencyMs = latencyMs,
accelerator = lastMessage.accelerator,
) )
newMessages.removeAt(newMessages.size - 1) newMessages.removeAt(newMessages.size - 1)
newMessages.add(newLastMessage) newMessages.add(newLastMessage)

View file

@ -23,7 +23,6 @@ import android.graphics.Bitmap
import android.graphics.BitmapFactory import android.graphics.BitmapFactory
import android.graphics.Matrix import android.graphics.Matrix
import android.net.Uri import android.net.Uri
import android.util.Log
import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.PickVisualMediaRequest import androidx.activity.result.PickVisualMediaRequest
import androidx.activity.result.contract.ActivityResultContracts import androidx.activity.result.contract.ActivityResultContracts
@ -31,9 +30,9 @@ import androidx.annotation.StringRes
import androidx.camera.core.CameraControl import androidx.camera.core.CameraControl
import androidx.camera.core.CameraSelector import androidx.camera.core.CameraSelector
import androidx.camera.core.ImageCapture import androidx.camera.core.ImageCapture
import androidx.camera.core.ImageProxy
import androidx.camera.lifecycle.ProcessCameraProvider import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.camera.lifecycle.awaitInstance import androidx.camera.lifecycle.awaitInstance
import androidx.camera.view.LifecycleCameraController
import androidx.camera.view.PreviewView import androidx.camera.view.PreviewView
import androidx.compose.foundation.Image import androidx.compose.foundation.Image
import androidx.compose.foundation.background import androidx.compose.foundation.background
@ -57,12 +56,12 @@ import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.automirrored.rounded.Send import androidx.compose.material.icons.automirrored.rounded.Send
import androidx.compose.material.icons.rounded.Add import androidx.compose.material.icons.rounded.Add
import androidx.compose.material.icons.rounded.Close import androidx.compose.material.icons.rounded.Close
import androidx.compose.material.icons.rounded.FlipCameraAndroid
import androidx.compose.material.icons.rounded.History import androidx.compose.material.icons.rounded.History
import androidx.compose.material.icons.rounded.Photo import androidx.compose.material.icons.rounded.Photo
import androidx.compose.material.icons.rounded.PhotoCamera import androidx.compose.material.icons.rounded.PhotoCamera
import androidx.compose.material.icons.rounded.PostAdd import androidx.compose.material.icons.rounded.PostAdd
import androidx.compose.material.icons.rounded.Stop import androidx.compose.material.icons.rounded.Stop
import androidx.compose.material3.Button
import androidx.compose.material3.DropdownMenu import androidx.compose.material3.DropdownMenu
import androidx.compose.material3.DropdownMenuItem import androidx.compose.material3.DropdownMenuItem
import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.ExperimentalMaterial3Api
@ -81,13 +80,13 @@ import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.runtime.setValue import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha import androidx.compose.ui.draw.alpha
import androidx.compose.ui.draw.clip import androidx.compose.ui.draw.clip
import androidx.compose.ui.draw.shadow import androidx.compose.ui.draw.shadow
import androidx.compose.ui.focus.focusModifier
import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalContext
@ -96,13 +95,13 @@ import androidx.compose.ui.tooling.preview.Preview
import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView import androidx.compose.ui.viewinterop.AndroidView
import androidx.core.content.ContextCompat import androidx.core.content.ContextCompat
import androidx.lifecycle.LifecycleOwner
import androidx.lifecycle.compose.LocalLifecycleOwner import androidx.lifecycle.compose.LocalLifecycleOwner
import com.google.ai.edge.gallery.R import com.google.ai.edge.gallery.R
import com.google.ai.edge.gallery.ui.common.createTempPictureUri import com.google.ai.edge.gallery.ui.common.createTempPictureUri
import com.google.ai.edge.gallery.ui.modelmanager.ModelManagerViewModel import com.google.ai.edge.gallery.ui.modelmanager.ModelManagerViewModel
import com.google.ai.edge.gallery.ui.preview.PreviewModelManagerViewModel import com.google.ai.edge.gallery.ui.preview.PreviewModelManagerViewModel
import com.google.ai.edge.gallery.ui.theme.GalleryTheme import com.google.ai.edge.gallery.ui.theme.GalleryTheme
import kotlinx.coroutines.launch
import java.util.concurrent.Executors import java.util.concurrent.Executors
/** /**
@ -131,6 +130,7 @@ fun MessageInputText(
showStopButtonWhenInProgress: Boolean = false, showStopButtonWhenInProgress: Boolean = false,
) { ) {
val context = LocalContext.current val context = LocalContext.current
val scope = rememberCoroutineScope()
val modelManagerUiState by modelManagerViewModel.uiState.collectAsState() val modelManagerUiState by modelManagerViewModel.uiState.collectAsState()
var showAddContentMenu by remember { mutableStateOf(false) } var showAddContentMenu by remember { mutableStateOf(false) }
var showTextInputHistorySheet by remember { mutableStateOf(false) } var showTextInputHistorySheet by remember { mutableStateOf(false) }
@ -144,6 +144,11 @@ fun MessageInputText(
newPickedImages.add(bitmap) newPickedImages.add(bitmap)
pickedImages = newPickedImages.toList() pickedImages = newPickedImages.toList()
} }
var hasFrontCamera by remember { mutableStateOf(false) }
LaunchedEffect(Unit) {
checkFrontCamera(context = context, callback = { hasFrontCamera = it })
}
// launches camera // launches camera
val cameraLauncher = val cameraLauncher =
@ -167,8 +172,8 @@ fun MessageInputText(
if (permissionGranted) { if (permissionGranted) {
showAddContentMenu = false showAddContentMenu = false
tempPhotoUri = context.createTempPictureUri() tempPhotoUri = context.createTempPictureUri()
// showCameraCaptureBottomSheet = true showCameraCaptureBottomSheet = true
cameraLauncher.launch(tempPhotoUri) // cameraLauncher.launch(tempPhotoUri)
} }
} }
@ -264,8 +269,8 @@ fun MessageInputText(
) -> { ) -> {
showAddContentMenu = false showAddContentMenu = false
tempPhotoUri = context.createTempPictureUri() tempPhotoUri = context.createTempPictureUri()
// showCameraCaptureBottomSheet = true showCameraCaptureBottomSheet = true
cameraLauncher.launch(tempPhotoUri) // cameraLauncher.launch(tempPhotoUri)
} }
// Otherwise, ask for permission // Otherwise, ask for permission
@ -408,12 +413,14 @@ fun MessageInputText(
ModalBottomSheet( ModalBottomSheet(
sheetState = cameraCaptureSheetState, sheetState = cameraCaptureSheetState,
onDismissRequest = { showCameraCaptureBottomSheet = false }) { onDismissRequest = { showCameraCaptureBottomSheet = false }) {
val lifecycleOwner = LocalLifecycleOwner.current val lifecycleOwner = LocalLifecycleOwner.current
val previewUseCase = remember { androidx.camera.core.Preview.Builder().build() } val previewUseCase = remember { androidx.camera.core.Preview.Builder().build() }
val imageCaptureUseCase = remember { ImageCapture.Builder().build() } val imageCaptureUseCase = remember { ImageCapture.Builder().build() }
var cameraProvider by remember { mutableStateOf<ProcessCameraProvider?>(null) } var cameraProvider by remember { mutableStateOf<ProcessCameraProvider?>(null) }
var cameraControl by remember { mutableStateOf<CameraControl?>(null) } var cameraControl by remember { mutableStateOf<CameraControl?>(null) }
val localContext = LocalContext.current val localContext = LocalContext.current
var cameraSide by remember { mutableStateOf(CameraSelector.LENS_FACING_BACK) }
val executor = remember { Executors.newSingleThreadExecutor() } val executor = remember { Executors.newSingleThreadExecutor() }
val capturedImageUri = remember { mutableStateOf<Uri?>(null) } val capturedImageUri = remember { mutableStateOf<Uri?>(null) }
@ -421,7 +428,7 @@ fun MessageInputText(
fun rebindCameraProvider() { fun rebindCameraProvider() {
cameraProvider?.let { cameraProvider -> cameraProvider?.let { cameraProvider ->
val cameraSelector = CameraSelector.Builder() val cameraSelector = CameraSelector.Builder()
.requireLensFacing(CameraSelector.LENS_FACING_FRONT) .requireLensFacing(cameraSide)
.build() .build()
cameraProvider.unbindAll() cameraProvider.unbindAll()
val camera = cameraProvider.bindToLifecycle( val camera = cameraProvider.bindToLifecycle(
@ -439,6 +446,10 @@ fun MessageInputText(
rebindCameraProvider() rebindCameraProvider()
} }
LaunchedEffect(cameraSide) {
rebindCameraProvider()
}
// val cameraController = remember { // val cameraController = remember {
// LifecycleCameraController(context).apply { // LifecycleCameraController(context).apply {
// bindToLifecycle(lifecycleOwner) // bindToLifecycle(lifecycleOwner)
@ -465,25 +476,92 @@ fun MessageInputText(
// cameraController.unbind() // Unbinds the camera to free up resources // cameraController.unbind() // Unbinds the camera to free up resources
// } // }
) )
// Close button.
IconButton(
onClick = {
scope.launch {
cameraCaptureSheetState.hide()
showCameraCaptureBottomSheet = false
}
}, colors = IconButtonDefaults.iconButtonColors(
containerColor = MaterialTheme.colorScheme.surfaceVariant,
), modifier = Modifier
.offset(x = (-8).dp, y = 8.dp)
.align(Alignment.TopEnd)
) {
Icon(
Icons.Rounded.Close,
contentDescription = "",
tint = MaterialTheme.colorScheme.primary
)
}
// Button that triggers the image capture process // Button that triggers the image capture process
IconButton( IconButton(
colors = IconButtonDefaults.iconButtonColors( colors = IconButtonDefaults.iconButtonColors(
containerColor = MaterialTheme.colorScheme.tertiaryContainer, containerColor = MaterialTheme.colorScheme.primary,
), ),
modifier = Modifier modifier = Modifier
.align(Alignment.BottomCenter) .align(Alignment.BottomCenter)
.padding(bottom = 32.dp) .padding(bottom = 32.dp)
.size(64.dp), .size(64.dp)
.border(2.dp, MaterialTheme.colorScheme.onPrimary, CircleShape),
onClick = { onClick = {
scope.launch {
val callback = object : ImageCapture.OnImageCapturedCallback() {
override fun onCaptureSuccess(image: ImageProxy) {
var bitmap = image.toBitmap()
val rotation = image.imageInfo.rotationDegrees
bitmap = if (rotation != 0) {
val matrix = Matrix().apply {
postRotate(rotation.toFloat())
}
Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true)
} else bitmap
updatePickedImages(bitmap)
image.close()
}
}
imageCaptureUseCase.takePicture(executor, callback)
cameraCaptureSheetState.hide()
showCameraCaptureBottomSheet = false
}
}, },
) { ) {
Icon( Icon(
Icons.Rounded.PhotoCamera, Icons.Rounded.PhotoCamera,
contentDescription = "", contentDescription = "",
tint = MaterialTheme.colorScheme.onTertiaryContainer, tint = MaterialTheme.colorScheme.onPrimary,
modifier = Modifier.size(36.dp) modifier = Modifier.size(36.dp)
) )
} }
// Button that toggles the front and back camera.
if (hasFrontCamera) {
IconButton(
colors = IconButtonDefaults.iconButtonColors(
containerColor = MaterialTheme.colorScheme.secondaryContainer,
),
modifier = Modifier
.align(Alignment.BottomEnd)
.padding(bottom = 40.dp, end = 32.dp)
.size(48.dp),
onClick = {
cameraSide = when (cameraSide) {
CameraSelector.LENS_FACING_BACK -> CameraSelector.LENS_FACING_FRONT
else -> CameraSelector.LENS_FACING_BACK
}
},
) {
Icon(
Icons.Rounded.FlipCameraAndroid,
contentDescription = "",
tint = MaterialTheme.colorScheme.onSecondaryContainer,
modifier = Modifier.size(24.dp)
)
}
}
} }
} }
} }
@ -501,13 +579,7 @@ private fun handleImageSelected(
val bitmap: Bitmap? = try { val bitmap: Bitmap? = try {
val inputStream = context.contentResolver.openInputStream(uri) val inputStream = context.contentResolver.openInputStream(uri)
val tmpBitmap = BitmapFactory.decodeStream(inputStream) val tmpBitmap = BitmapFactory.decodeStream(inputStream)
if (rotateForPortrait && tmpBitmap.width > tmpBitmap.height) { rotateImageIfNecessary(bitmap = tmpBitmap, rotateForPortrait = rotateForPortrait)
val matrix = Matrix()
matrix.postRotate(90f)
Bitmap.createBitmap(tmpBitmap, 0, 0, tmpBitmap.width, tmpBitmap.height, matrix, true)
} else {
tmpBitmap
}
} catch (e: Exception) { } catch (e: Exception) {
e.printStackTrace() e.printStackTrace()
null null
@ -517,6 +589,31 @@ private fun handleImageSelected(
} }
} }
private fun rotateImageIfNecessary(bitmap: Bitmap, rotateForPortrait: Boolean = false): Bitmap {
return if (rotateForPortrait && bitmap.width > bitmap.height) {
val matrix = Matrix()
matrix.postRotate(90f)
Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true)
} else {
bitmap
}
}
private fun checkFrontCamera(context: Context, callback: (Boolean) -> Unit) {
val cameraProviderFuture = ProcessCameraProvider.getInstance(context)
cameraProviderFuture.addListener(Runnable {
val cameraProvider = cameraProviderFuture.get()
try {
// Attempt to select the default front camera
val hasFront = cameraProvider.hasCamera(CameraSelector.DEFAULT_FRONT_CAMERA)
callback(hasFront)
} catch (e: Exception) {
e.printStackTrace()
callback(false)
}
}, ContextCompat.getMainExecutor(context))
}
private fun createMessagesToSend(pickedImages: List<Bitmap>, text: String): List<ChatMessage> { private fun createMessagesToSend(pickedImages: List<Bitmap>, text: String): List<ChatMessage> {
val messages: MutableList<ChatMessage> = mutableListOf() val messages: MutableList<ChatMessage> = mutableListOf()
if (pickedImages.isNotEmpty()) { if (pickedImages.isNotEmpty()) {

View file

@ -180,6 +180,9 @@ private fun getMessageLayoutConfig(
horizontalArrangement = Arrangement.SpaceBetween horizontalArrangement = Arrangement.SpaceBetween
modifier = modifier.fillMaxWidth() modifier = modifier.fillMaxWidth()
userLabel = "Stats" userLabel = "Stats"
if (message.accelerator.isNotEmpty()) {
userLabel = "${userLabel} on ${message.accelerator}"
}
} }
is ChatMessageImageWithHistory -> { is ChatMessageImageWithHistory -> {

View file

@ -388,7 +388,7 @@ private fun TaskList(
introText, introText,
textAlign = TextAlign.Center, textAlign = TextAlign.Center,
style = MaterialTheme.typography.bodyLarge.copy(fontWeight = FontWeight.SemiBold), style = MaterialTheme.typography.bodyLarge.copy(fontWeight = FontWeight.SemiBold),
modifier = Modifier.padding(bottom = 20.dp) modifier = Modifier.padding(bottom = 20.dp).padding(horizontal = 16.dp)
) )
} }

View file

@ -20,6 +20,7 @@ import android.content.Context
import android.graphics.Bitmap import android.graphics.Bitmap
import android.util.Log import android.util.Log
import androidx.lifecycle.viewModelScope import androidx.lifecycle.viewModelScope
import com.google.ai.edge.gallery.data.ConfigKey
import com.google.ai.edge.gallery.data.Model import com.google.ai.edge.gallery.data.Model
import com.google.ai.edge.gallery.data.TASK_LLM_CHAT import com.google.ai.edge.gallery.data.TASK_LLM_CHAT
import com.google.ai.edge.gallery.data.TASK_LLM_ASK_IMAGE import com.google.ai.edge.gallery.data.TASK_LLM_ASK_IMAGE
@ -47,6 +48,7 @@ private val STATS = listOf(
open class LlmChatViewModel(curTask: Task = TASK_LLM_CHAT) : ChatViewModel(task = curTask) { open class LlmChatViewModel(curTask: Task = TASK_LLM_CHAT) : ChatViewModel(task = curTask) {
fun generateResponse(model: Model, input: String, image: Bitmap? = null, onError: () -> Unit) { fun generateResponse(model: Model, input: String, image: Bitmap? = null, onError: () -> Unit) {
val accelerator = model.getStringConfigValue(key = ConfigKey.ACCELERATOR, defaultValue = "")
viewModelScope.launch(Dispatchers.Default) { viewModelScope.launch(Dispatchers.Default) {
setInProgress(true) setInProgress(true)
setPreparing(true) setPreparing(true)
@ -54,7 +56,7 @@ open class LlmChatViewModel(curTask: Task = TASK_LLM_CHAT) : ChatViewModel(task
// Loading. // Loading.
addMessage( addMessage(
model = model, model = model,
message = ChatMessageLoading(), message = ChatMessageLoading(accelerator = accelerator),
) )
// Wait for instance to be initialized. // Wait for instance to be initialized.
@ -103,7 +105,12 @@ open class LlmChatViewModel(curTask: Task = TASK_LLM_CHAT) : ChatViewModel(task
// Add an empty message that will receive streaming results. // Add an empty message that will receive streaming results.
addMessage( addMessage(
model = model, message = ChatMessageText(content = "", side = ChatSide.AGENT) model = model,
message = ChatMessageText(
content = "",
side = ChatSide.AGENT,
accelerator = accelerator
)
) )
} }
@ -133,6 +140,7 @@ open class LlmChatViewModel(curTask: Task = TASK_LLM_CHAT) : ChatViewModel(task
), ),
running = false, running = false,
latencyMs = -1f, latencyMs = -1f,
accelerator = accelerator,
) )
) )
} }

View file

@ -28,6 +28,8 @@ import androidx.compose.foundation.layout.size
import androidx.compose.foundation.pager.HorizontalPager import androidx.compose.foundation.pager.HorizontalPager
import androidx.compose.foundation.pager.rememberPagerState import androidx.compose.foundation.pager.rememberPagerState
import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.text.BasicText
import androidx.compose.foundation.text.TextAutoSize
import androidx.compose.foundation.verticalScroll import androidx.compose.foundation.verticalScroll
import androidx.compose.material.icons.Icons import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.outlined.AutoAwesome import androidx.compose.material.icons.outlined.AutoAwesome
@ -56,6 +58,8 @@ import androidx.compose.ui.graphics.Color
import androidx.compose.ui.platform.LocalClipboardManager import androidx.compose.ui.platform.LocalClipboardManager
import androidx.compose.ui.text.AnnotatedString import androidx.compose.ui.text.AnnotatedString
import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import com.google.ai.edge.gallery.data.ConfigKey
import com.google.ai.edge.gallery.data.Model import com.google.ai.edge.gallery.data.Model
import com.google.ai.edge.gallery.data.TASK_LLM_PROMPT_LAB import com.google.ai.edge.gallery.data.TASK_LLM_PROMPT_LAB
import com.google.ai.edge.gallery.ui.common.chat.MarkdownText import com.google.ai.edge.gallery.ui.common.chat.MarkdownText
@ -88,6 +92,7 @@ fun ResponsePanel(
val pagerState = rememberPagerState( val pagerState = rememberPagerState(
initialPage = task.models.indexOf(model), initialPage = task.models.indexOf(model),
pageCount = { task.models.size }) pageCount = { task.models.size })
val accelerator = model.getStringConfigValue(key = ConfigKey.ACCELERATOR, defaultValue = "")
// Select the "response" tab when prompt template changes. // Select the "response" tab when prompt template changes.
LaunchedEffect(selectedPromptTemplateType) { LaunchedEffect(selectedPromptTemplateType) {
@ -191,7 +196,22 @@ fun ResponsePanel(
.size(16.dp) .size(16.dp)
.alpha(0.7f) .alpha(0.7f)
) )
Text(text = title) var curTitle = title
if (accelerator.isNotEmpty()) {
curTitle = "$curTitle on $accelerator"
}
val titleColor = MaterialTheme.colorScheme.primary
BasicText(
text = curTitle,
maxLines = 1,
color = { titleColor },
style = MaterialTheme.typography.bodyMedium,
autoSize = TextAutoSize.StepBased(
minFontSize = 9.sp,
maxFontSize = 14.sp,
stepSize = 1.sp
)
)
} }
}) })
} }

View file

@ -27,6 +27,7 @@ import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.size
import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.items import androidx.compose.foundation.lazy.items
import androidx.compose.foundation.lazy.rememberLazyListState
import androidx.compose.material.icons.Icons import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.outlined.Code import androidx.compose.material.icons.outlined.Code
import androidx.compose.material.icons.outlined.Description import androidx.compose.material.icons.outlined.Description
@ -91,11 +92,14 @@ fun ModelList(
} }
} }
val listState = rememberLazyListState()
Box(contentAlignment = Alignment.BottomEnd) { Box(contentAlignment = Alignment.BottomEnd) {
LazyColumn( LazyColumn(
modifier = modifier.padding(top = 8.dp), modifier = modifier.padding(top = 8.dp),
contentPadding = contentPadding, contentPadding = contentPadding,
verticalArrangement = Arrangement.spacedBy(8.dp), verticalArrangement = Arrangement.spacedBy(8.dp),
state = listState,
) { ) {
// Headline. // Headline.
item(key = "headline") { item(key = "headline") {
@ -103,7 +107,9 @@ fun ModelList(
task.description, task.description,
textAlign = TextAlign.Center, textAlign = TextAlign.Center,
style = MaterialTheme.typography.bodyLarge.copy(fontWeight = FontWeight.SemiBold), style = MaterialTheme.typography.bodyLarge.copy(fontWeight = FontWeight.SemiBold),
modifier = Modifier.fillMaxWidth() modifier = Modifier
.fillMaxWidth()
.padding(horizontal = 16.dp),
) )
} }

View file

@ -382,6 +382,8 @@ open class ModelManagerViewModel(
} }
_uiState.update { _uiState.value.copy(textInputHistory = newHistory) } _uiState.update { _uiState.value.copy(textInputHistory = newHistory) }
dataStoreRepository.saveTextInputHistory(_uiState.value.textInputHistory) dataStoreRepository.saveTextInputHistory(_uiState.value.textInputHistory)
} else {
promoteTextInputHistoryItem(text)
} }
} }

View file

@ -16,10 +16,16 @@
package com.google.ai.edge.gallery.worker package com.google.ai.edge.gallery.worker
import android.app.NotificationChannel
import android.app.NotificationManager
import android.content.Context import android.content.Context
import android.content.pm.ServiceInfo
import android.os.Build
import android.util.Log import android.util.Log
import androidx.core.app.NotificationCompat
import androidx.work.CoroutineWorker import androidx.work.CoroutineWorker
import androidx.work.Data import androidx.work.Data
import androidx.work.ForegroundInfo
import androidx.work.WorkerParameters import androidx.work.WorkerParameters
import com.google.ai.edge.gallery.data.KEY_MODEL_DOWNLOAD_ACCESS_TOKEN import com.google.ai.edge.gallery.data.KEY_MODEL_DOWNLOAD_ACCESS_TOKEN
import com.google.ai.edge.gallery.data.KEY_MODEL_DOWNLOAD_APP_TS import com.google.ai.edge.gallery.data.KEY_MODEL_DOWNLOAD_APP_TS
@ -32,6 +38,7 @@ import com.google.ai.edge.gallery.data.KEY_MODEL_DOWNLOAD_REMAINING_MS
import com.google.ai.edge.gallery.data.KEY_MODEL_EXTRA_DATA_DOWNLOAD_FILE_NAMES import com.google.ai.edge.gallery.data.KEY_MODEL_EXTRA_DATA_DOWNLOAD_FILE_NAMES
import com.google.ai.edge.gallery.data.KEY_MODEL_EXTRA_DATA_URLS import com.google.ai.edge.gallery.data.KEY_MODEL_EXTRA_DATA_URLS
import com.google.ai.edge.gallery.data.KEY_MODEL_IS_ZIP import com.google.ai.edge.gallery.data.KEY_MODEL_IS_ZIP
import com.google.ai.edge.gallery.data.KEY_MODEL_NAME
import com.google.ai.edge.gallery.data.KEY_MODEL_START_UNZIPPING import com.google.ai.edge.gallery.data.KEY_MODEL_START_UNZIPPING
import com.google.ai.edge.gallery.data.KEY_MODEL_TOTAL_BYTES import com.google.ai.edge.gallery.data.KEY_MODEL_TOTAL_BYTES
import com.google.ai.edge.gallery.data.KEY_MODEL_UNZIPPED_DIR import com.google.ai.edge.gallery.data.KEY_MODEL_UNZIPPED_DIR
@ -57,14 +64,40 @@ data class UrlAndFileName(
val fileName: String, val fileName: String,
) )
private const val FOREGROUND_NOTIFICATION_CHANNEL_ID = "model_download_channel_foreground"
private var channelCreated = false
class DownloadWorker(context: Context, params: WorkerParameters) : class DownloadWorker(context: Context, params: WorkerParameters) :
CoroutineWorker(context, params) { CoroutineWorker(context, params) {
private val externalFilesDir = context.getExternalFilesDir(null) private val externalFilesDir = context.getExternalFilesDir(null)
private val notificationManager =
context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
// Unique notification id.
private val notificationId: Int = params.id.hashCode()
init {
if (!channelCreated) {
// Create a notification channel for showing notifications for model downloading progress.
val channel = NotificationChannel(
FOREGROUND_NOTIFICATION_CHANNEL_ID,
"Model Downloading",
// Make it silent.
NotificationManager.IMPORTANCE_LOW
).apply {
description = "Notifications for model downloading"
}
notificationManager.createNotificationChannel(channel)
channelCreated = true
}
}
override suspend fun doWork(): Result { override suspend fun doWork(): Result {
val appTs = readLaunchInfo(context = applicationContext)?.ts ?: 0 val appTs = readLaunchInfo(context = applicationContext)?.ts ?: 0
val fileUrl = inputData.getString(KEY_MODEL_URL) val fileUrl = inputData.getString(KEY_MODEL_URL)
val modelName = inputData.getString(KEY_MODEL_NAME) ?: "Model"
val version = inputData.getString(KEY_MODEL_VERSION)!! val version = inputData.getString(KEY_MODEL_VERSION)!!
val fileName = inputData.getString(KEY_MODEL_DOWNLOAD_FILE_NAME) val fileName = inputData.getString(KEY_MODEL_DOWNLOAD_FILE_NAME)
val modelDir = inputData.getString(KEY_MODEL_DOWNLOAD_MODEL_DIR)!! val modelDir = inputData.getString(KEY_MODEL_DOWNLOAD_MODEL_DIR)!!
@ -87,6 +120,9 @@ class DownloadWorker(context: Context, params: WorkerParameters) :
Result.failure() Result.failure()
} else { } else {
return@withContext try { return@withContext try {
// Set the worker as a foreground service immediately.
setForeground(createForegroundInfo(progress = 0, modelName = modelName))
// Collect data for all files. // Collect data for all files.
val allFiles: MutableList<UrlAndFileName> = mutableListOf() val allFiles: MutableList<UrlAndFileName> = mutableListOf()
allFiles.add(UrlAndFileName(url = fileUrl, fileName = fileName)) allFiles.add(UrlAndFileName(url = fileUrl, fileName = fileName))
@ -206,6 +242,11 @@ class DownloadWorker(context: Context, params: WorkerParameters) :
KEY_MODEL_DOWNLOAD_REMAINING_MS, remainingMs.toLong() KEY_MODEL_DOWNLOAD_REMAINING_MS, remainingMs.toLong()
).build() ).build()
) )
setForeground(
createForegroundInfo(
progress = (downloadedBytes * 100 / totalBytes).toInt(), modelName = modelName
)
)
Log.d(TAG, "downloadedBytes: $downloadedBytes") Log.d(TAG, "downloadedBytes: $downloadedBytes")
lastSetProgressTs = curTs lastSetProgressTs = curTs
} }
@ -221,11 +262,10 @@ class DownloadWorker(context: Context, params: WorkerParameters) :
setProgress(Data.Builder().putBoolean(KEY_MODEL_START_UNZIPPING, true).build()) setProgress(Data.Builder().putBoolean(KEY_MODEL_START_UNZIPPING, true).build())
// Prepare target dir. // Prepare target dir.
val destDir = val destDir = File(
File( externalFilesDir,
externalFilesDir, listOf(modelDir, version, unzippedDir).joinToString(File.separator)
listOf(modelDir, version, unzippedDir).joinToString(File.separator) )
)
if (!destDir.exists()) { if (!destDir.exists()) {
destDir.mkdirs() destDir.mkdirs()
} }
@ -276,4 +316,42 @@ class DownloadWorker(context: Context, params: WorkerParameters) :
} }
} }
} }
override suspend fun getForegroundInfo(): ForegroundInfo {
// Initial progress is 0
return createForegroundInfo(0)
}
/**
* Creates a [ForegroundInfo] object for the download worker's ongoing notification.
* This notification is used to keep the worker running in the foreground, indicating
* to the user that an active download is in progress.
*/
private fun createForegroundInfo(progress: Int, modelName: String? = null): ForegroundInfo {
// Create a notification for the foreground service
var title = "Downloading model"
if (modelName != null) {
title = "Downloading \"$modelName\""
}
val content = "Downloading in progress: $progress%"
val notification =
NotificationCompat.Builder(applicationContext, FOREGROUND_NOTIFICATION_CHANNEL_ID)
.setContentTitle(title).setContentText(content)
.setSmallIcon(android.R.drawable.ic_dialog_info)
.setOngoing(true) // Makes the notification non-dismissable
.setProgress(100, progress, false) // Show progress
.build()
return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
ForegroundInfo(
notificationId, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC
)
} else {
ForegroundInfo(
notificationId,
notification,
)
}
}
} }

View file

@ -1,20 +1,4 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<!--
Copyright 2025 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android"> <adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
<background android:drawable="@mipmap/ic_launcher_background"/> <background android:drawable="@mipmap/ic_launcher_background"/>
<foreground android:drawable="@mipmap/ic_launcher_foreground"/> <foreground android:drawable="@mipmap/ic_launcher_foreground"/>

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.4 KiB

After

Width:  |  Height:  |  Size: 4.2 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 855 B

After

Width:  |  Height:  |  Size: 852 B

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.4 KiB

After

Width:  |  Height:  |  Size: 4.5 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.4 KiB

After

Width:  |  Height:  |  Size: 4.5 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.3 KiB

After

Width:  |  Height:  |  Size: 2.7 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 463 B

After

Width:  |  Height:  |  Size: 459 B

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.9 KiB

After

Width:  |  Height:  |  Size: 3 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.9 KiB

After

Width:  |  Height:  |  Size: 3 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.9 KiB

After

Width:  |  Height:  |  Size: 5.9 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.3 KiB

After

Width:  |  Height:  |  Size: 1.3 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6 KiB

After

Width:  |  Height:  |  Size: 6.5 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6 KiB

After

Width:  |  Height:  |  Size: 6.5 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 13 KiB

After

Width:  |  Height:  |  Size: 9.2 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.9 KiB

After

Width:  |  Height:  |  Size: 2.9 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 10 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 10 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 13 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.1 KiB

After

Width:  |  Height:  |  Size: 4.1 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

Before After
Before After