From d5767156b9296c2fd9e8260030376278cf821a4c Mon Sep 17 00:00:00 2001 From: gcw_4spBpAfv Date: Thu, 5 Mar 2026 18:50:48 +0800 Subject: [PATCH] ai state machine init --- .vscode/settings.json | 3 + app/note/design_doc | 80 +++- .../com/digitalperson/Live2DChatActivity.kt | 373 ++++++++++++------ .../digitalperson/cloud/CloudApiManager.java | 7 + .../face/FaceDetectionPipeline.kt | 71 ++-- .../digitalperson/face/FaceFeatureStore.kt | 16 +- .../com/digitalperson/face/FaceRecognizer.kt | 57 ++- .../DigitalHumanInteractionController.kt | 268 +++++++++++++ .../interaction/UserMemoryStore.kt | 177 +++++++++ .../com/digitalperson/ui/Live2DUiManager.kt | 20 +- 10 files changed, 855 insertions(+), 217 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 app/src/main/java/com/digitalperson/interaction/DigitalHumanInteractionController.kt create mode 100644 app/src/main/java/com/digitalperson/interaction/UserMemoryStore.kt diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..e0f15db --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "java.configuration.updateBuildConfiguration": "automatic" +} \ No newline at end of file diff --git a/app/note/design_doc b/app/note/design_doc index f971813..6a4d067 100644 --- a/app/note/design_doc +++ b/app/note/design_doc @@ -113,7 +113,85 @@ https://www.modelscope.cn/datasets/shaoxuan/WIDER_FACE/files 8. 人脸识别模型是insightface的r18模型，转成了rknn格式，并且使用了 lfw 的数据集进行了校准，下载地址： https://tianchi.aliyun.com/dataset/93864 -9. +9. 本地LLM，用的是 rkllm 模型，由于内存限制，只能用较小的模型，如 Qwen3-0.6B-rk3588-w8a8.rkllm。 + +10. 数字人交互设计说明书（优化版） +核心原则 +* 尊重用户：给用户足够的反应时间，不抢话，不催促。 +* 主动但不打扰：用户沉默时适度引导，但不强行交互；数字人拥有独立的内心活动，即便无人交互也保持“生命感”。 +* 个性化：记住每个用户的偏好、名字、对话历史，并据此调整问候和回应。 +* 动作同步：Live2D人物的表情与动作与当前状态、情绪、内心活动相匹配。 +* 持续内心活动：数字人即使在没有用户时也会进行“回忆”或“思考”，并通过文字展现，用户可随时打断并询问想法。 + +状态定义 +[空闲状态]：无人，数字人无内心活动（省电模式）。 +[回忆状态]：无人，数字人正在根据记忆产生内心想法，通过表情表现。 +[问候状态]：检测到用户，主动打招呼。 +[等待回复状态]：问候后或对话间隙，等待用户回复。 +[对话状态]：正在与用户交流，根据情绪调整动作。 +[主动引导状态]：用户沉默但仍在画面中，数字人主动开启新话题。 +[告别状态]：用户离开画面，数字人告别。 +[空闲状态] + 动作 ： haru_g_idle （待机动作） + ↓ +检测到人脸 → 等待1秒（让人脸稳定） + ↓ +[问候状态] + 动作 ： + - 认识用户： haru_g_m22 （高兴动作） + - 不认识： haru_g_m01（中性动作） + AI问候（根据是否认识个性化） + 启动20秒计时器 + ↓ +[等待回复状态] + 动作 ： haru_g_m17 + ↓ + ├─ 如果用户在20秒内回复 → 进入[对话状态] + │ 动作 ：根据对话情绪动态调整 + │ - 开心： haru_g_m22 / haru_g_m21 / haru_g_m18 / haru_g_m09 / haru_g_m08 + │ - 悲伤： haru_g_m25 / haru_g_m24 / haru_g_m05 / haru_g_m16 + │ - 惊讶： haru_g_m26 / haru_g_m12 + │ - 愤怒： haru_g_m04 / haru_g_m11 / haru_g_m04 + │ - 平静： haru_g_m15 / haru_g_m07 / haru_g_m06 / haru_g_m02 / haru_g_m01 + │ AI根据内容回应，持续直到用户说“再见” + │ ↓ + │ （用户离开）→ 进入[告别状态] + │ + └─ 如果20秒内无回复 + ↓ + 检查用户是否还在画面 + ↓ + ├─ 如果还在 → [主动引导状态] + │ 动作 ： haru_g_m15 / haru_g_m07 （中性动作） + │ AI开启轻松话题（如“我出一道数学题考考你吧？1+6等于多少？”） + │ 等待10秒 + │ ↓ + │ ├─ 回复 → 进入[对话状态] + │ └─ 没回复 → 换话题，如，我们上完厕所应该干什么呀？最多重复3次 + │ 动作 ： haru_g_m22 / haru_g_m18 + │ （重复尝试）→ 再次等待10秒 + │ （3次后仍无回复）→ 回到[等待回复状态] + │ + └─ 如果已离开 → [告别状态] + 动作 ： haru_g_idle （告别后待机） + ↓ + 3秒后 → 回到[空闲状态] + +[空闲状态] （长时间无人） + ↓ + 如果持续30秒无人 → 进入[回忆状态] + 动作 ： haru_g_m15 + 回顾之前的聊天记录，生成想法，不发声，把想法显示在屏幕上 + ↓ + 如果检测到人脸 → 立即中断回忆，进入[问候状态] +[回忆状态] 被用户询问“你在想什么？” + ↓ + AI说出最近的想法（如“我刚才在想，上次你说你喜欢蓝色...”） + ↓ + 进入[问候状态]或直接进入[对话状态]（根据用户后续反应） + +[告别状态] （用户离开后） + 动作 ： 简短告别，用语音，→ 等待3秒 → 回到[空闲状态] diff --git a/app/src/main/java/com/digitalperson/Live2DChatActivity.kt b/app/src/main/java/com/digitalperson/Live2DChatActivity.kt index 3961625..dbadf38 100644 --- a/app/src/main/java/com/digitalperson/Live2DChatActivity.kt +++ b/app/src/main/java/com/digitalperson/Live2DChatActivity.kt @@ -27,10 +27,15 @@ import com.digitalperson.face.ImageProxyBitmapConverter import com.digitalperson.metrics.TraceManager import com.digitalperson.metrics.TraceSession import com.digitalperson.tts.TtsController +import com.digitalperson.interaction.DigitalHumanInteractionController +import com.digitalperson.interaction.InteractionActionHandler +import com.digitalperson.interaction.InteractionState +import com.digitalperson.interaction.UserMemoryStore import com.digitalperson.llm.LLMManager import com.digitalperson.llm.LLMManagerCallback import com.digitalperson.util.FileHelper import java.io.File +import org.json.JSONObject import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import kotlinx.coroutines.CoroutineScope @@ -85,9 +90,18 @@ class Live2DChatActivity : AppCompatActivity() { private lateinit var cameraPreviewView: PreviewView private lateinit var faceOverlayView: FaceOverlayView private lateinit var faceDetectionPipeline: FaceDetectionPipeline + private lateinit var interactionController: DigitalHumanInteractionController + private lateinit var userMemoryStore: UserMemoryStore private var facePipelineReady: Boolean = false private var cameraProvider: ProcessCameraProvider? = null private lateinit var cameraAnalyzerExecutor: ExecutorService + private var activeUserId: String = "guest" + private var pendingLocalThoughtCallback: ((String) -> Unit)? = null + private var pendingLocalProfileCallback: ((String) -> Unit)? = null + private var localThoughtSilentMode: Boolean = false + private val recentConversationLines = ArrayList() + private var recentConversationDirty: Boolean = false + private var lastFacePresent: Boolean = false override fun onRequestPermissionsResult( requestCode: Int, @@ -143,15 +157,73 @@ class Live2DChatActivity : AppCompatActivity() { cameraPreviewView.implementationMode = PreviewView.ImplementationMode.COMPATIBLE faceOverlayView = findViewById(R.id.face_overlay) cameraAnalyzerExecutor = Executors.newSingleThreadExecutor() + userMemoryStore = UserMemoryStore(applicationContext) + interactionController = DigitalHumanInteractionController( + scope = ioScope, + handler = object : InteractionActionHandler { + override fun onStateChanged(state: InteractionState) { + runOnUiThread { + uiManager.appendToUi("\n[State] $state\n") + } + if (state == InteractionState.IDLE) { + analyzeUserProfileInIdleIfNeeded() + } + } + + override fun playMotion(motionName: String) { + playInteractionMotion(motionName) + } + + override fun appendText(text: String) { + uiManager.appendToUi(text) + } + + override fun speak(text: String) { + ttsController.enqueueSegment(text) + ttsController.enqueueEnd() + } + + override fun requestCloudReply(userText: String) { + llmInFlight = true + Log.i(TAG_LLM, "Routing dialogue to CLOUD") + cloudApiManager.callLLM(buildCloudPromptWithUserProfile(userText)) + } + + override fun requestLocalThought(prompt: String, onResult: (String) -> Unit) { + this@Live2DChatActivity.requestLocalThought(prompt, onResult) + } + + override fun onRememberUser(faceIdentityId: String, name: String?) { + activeUserId = faceIdentityId + userMemoryStore.upsertUserSeen(activeUserId, name) + } + + override fun saveThought(thought: String) { + userMemoryStore.upsertUserSeen(activeUserId, null) + userMemoryStore.updateThought(activeUserId, thought) + } + + override fun loadLatestThought(): String? = userMemoryStore.getLatestThought() + + override fun addToChatHistory(role: String, content: String) { + appendConversationLine(role, content) + } + + override fun addAssistantMessageToCloudHistory(content: String) { + cloudApiManager.addAssistantMessage(content) + } + } + ) faceDetectionPipeline = FaceDetectionPipeline( context = applicationContext, onResult = { result -> faceOverlayView.updateResult(result) }, - onGreeting = { greeting -> - uiManager.appendToUi("\n[Face] $greeting\n") - ttsController.enqueueSegment(greeting) - ttsController.enqueueEnd() + onPresenceChanged = { present, faceIdentityId, recognizedName -> + if (present == lastFacePresent) return@FaceDetectionPipeline + lastFacePresent = present + Log.d(TAG_ACTIVITY, "present=$present, faceIdentityId=$faceIdentityId, recognized=$recognizedName") + interactionController.onFacePresenceChanged(present, faceIdentityId, recognizedName) } ) @@ -200,13 +272,12 @@ class Live2DChatActivity : AppCompatActivity() { // 设置 LLM 模式开关 uiManager.setLLMSwitchListener { isChecked -> + // 交互状态机固定路由：用户对话走云端，回忆走本地。此开关仅作为本地LLM可用性提示。 useLocalLLM = isChecked - Log.i(TAG_LLM, "LLM mode switched: useLocalLLM=$useLocalLLM") - uiManager.showToast("LLM模式已切换到${if (isChecked) "本地" else "云端"}") - // 重新初始化 LLM - initLLM() + uiManager.showToast("状态机路由已固定：对话云端，回忆本地") } // 默认不显示 LLM 开关，等模型下载完成后再显示 + uiManager.showLLMSwitch(false) if (AppConfig.USE_HOLD_TO_SPEAK) { uiManager.setButtonsEnabled(recordEnabled = false) @@ -226,8 +297,9 @@ class Live2DChatActivity : AppCompatActivity() { vadManager = VadManager(this) vadManager.setCallback(createVadCallback()) - // 初始化 LLM 管理器 + // 初始化本地 LLM（用于 memory 状态） initLLM() + interactionController.start() // 检查是否需要下载模型 if (!FileHelper.isLocalLLMAvailable(this)) { @@ -259,8 +331,8 @@ class Live2DChatActivity : AppCompatActivity() { if (FileHelper.isLocalLLMAvailable(this)) { Log.i(AppConfig.TAG, "Local LLM is available, enabling local LLM switch") // 显示本地 LLM 开关，并同步状态 - uiManager.showLLMSwitch(true) - uiManager.setLLMSwitchChecked(useLocalLLM) + uiManager.showLLMSwitch(false) + initLLM() } } else { Log.e(AppConfig.TAG, "Failed to download model files: $message") @@ -275,8 +347,7 @@ class Live2DChatActivity : AppCompatActivity() { // 模型已存在，直接初始化其他组件 initializeOtherComponents() // 显示本地 LLM 开关，并同步状态 - uiManager.showLLMSwitch(true) - uiManager.setLLMSwitchChecked(useLocalLLM) + uiManager.showLLMSwitch(false) } } @@ -348,6 +419,7 @@ class Live2DChatActivity : AppCompatActivity() { runOnUiThread { uiManager.appendToUi("\n\n[ASR] ${text}\n") } + appendConversationLine("用户", text) currentTrace?.markRecordingDone() currentTrace?.markLlmResponseReceived() } @@ -361,17 +433,8 @@ class Live2DChatActivity : AppCompatActivity() { override fun isLlmInFlight(): Boolean = llmInFlight override fun onLlmCalled(text: String) { - llmInFlight = true - Log.d(AppConfig.TAG, "Calling LLM with text: $text") - if (useLocalLLM) { - Log.i(TAG_LLM, "Routing to LOCAL LLM") - // 使用本地 LLM 生成回复 - generateResponse(text) - } else { - Log.i(TAG_LLM, "Routing to CLOUD LLM") - // 使用云端 LLM 生成回复 - cloudApiManager.callLLM(text) - } + Log.d(AppConfig.TAG, "Forward ASR text to interaction controller: $text") + interactionController.onUserAsrText(text) } } @@ -390,6 +453,7 @@ class Live2DChatActivity : AppCompatActivity() { override fun onLLMResponseReceived(response: String) { currentTrace?.markLlmDone() llmInFlight = false + appendConversationLine("助手", response) if (enableStreaming) { for (seg in segmenter.flush()) { @@ -411,6 +475,7 @@ class Live2DChatActivity : AppCompatActivity() { ttsController.enqueueSegment(filteredText) ttsController.enqueueEnd() } + interactionController.onDialogueResponseFinished() } override fun onLLMStreamingChunkReceived(chunk: String) { @@ -442,6 +507,7 @@ class Live2DChatActivity : AppCompatActivity() { override fun onError(errorMessage: String) { llmInFlight = false uiManager.showToast(errorMessage, Toast.LENGTH_LONG) + interactionController.onDialogueResponseFinished() onStopClicked(userInitiated = false) } } @@ -479,6 +545,7 @@ class Live2DChatActivity : AppCompatActivity() { override fun onDestroy() { super.onDestroy() + try { interactionController.stop() } catch (_: Throwable) {} stopCameraPreviewAndDetection() onStopClicked(userInitiated = false) ioScope.cancel() @@ -490,6 +557,7 @@ class Live2DChatActivity : AppCompatActivity() { try { cameraAnalyzerExecutor.shutdown() } catch (_: Throwable) {} try { ttsController.release() } catch (_: Throwable) {} try { llmManager?.destroy() } catch (_: Throwable) {} + try { userMemoryStore.close() } catch (_: Throwable) {} try { uiManager.release() } catch (_: Throwable) {} try { audioProcessor.release() } catch (_: Throwable) {} } @@ -757,85 +825,158 @@ class Live2DChatActivity : AppCompatActivity() { } Log.d(AppConfig.TAG, "processSamplesLoop stopped") } + + private fun playInteractionMotion(motionName: String) { + when (motionName) { + "haru_g_m22.motion3.json" -> uiManager.setMood("高兴") + "haru_g_m01.motion3.json", "haru_g_m17.motion3.json" -> uiManager.setMood("中性") + "haru_g_m15.motion3.json" -> uiManager.setMood("关心") + "haru_g_idle.motion3.json" -> uiManager.setMood("平和") + else -> uiManager.setMood("中性") + } + } + + private fun appendConversationLine(role: String, text: String) { + val line = "$role: ${text.trim()}" + if (line.length <= 4) return + recentConversationLines.add(line) + if (recentConversationLines.size > 12) { + recentConversationLines.removeAt(0) + } + recentConversationDirty = true + } + + private fun buildCloudPromptWithUserProfile(userText: String): String { + val profile = userMemoryStore.getMemory(activeUserId) ?: return userText + val profileParts = ArrayList() + profile.displayName?.takeIf { it.isNotBlank() }?.let { profileParts.add("姓名:$it") } + profile.age?.takeIf { it.isNotBlank() }?.let { profileParts.add("年龄:$it") } + profile.gender?.takeIf { it.isNotBlank() }?.let { profileParts.add("性别:$it") } + profile.hobbies?.takeIf { it.isNotBlank() }?.let { profileParts.add("爱好:$it") } + profile.profileSummary?.takeIf { it.isNotBlank() }?.let { profileParts.add("画像:$it") } + if (profileParts.isEmpty()) return userText + return buildString { + append("[用户画像]\n") + append(profileParts.joinToString("；")) + append("\n[/用户画像]\n") + append(userText) + } + } + + private fun analyzeUserProfileInIdleIfNeeded() { + if (!recentConversationDirty || !activeUserId.startsWith("face_")) return + if (recentConversationLines.isEmpty()) return + val dialogue = recentConversationLines.joinToString("\n") + requestLocalProfileExtraction(dialogue) { raw -> + try { + val json = parseFirstJsonObject(raw) + val name = json.optString("name", "").trim().ifBlank { null } + val age = json.optString("age", "").trim().ifBlank { null } + val gender = json.optString("gender", "").trim().ifBlank { null } + val hobbies = json.optString("hobbies", "").trim().ifBlank { null } + val summary = json.optString("summary", "").trim().ifBlank { null } + if (name != null) { + userMemoryStore.updateDisplayName(activeUserId, name) + } + userMemoryStore.updateProfile(activeUserId, age, gender, hobbies, summary) + recentConversationDirty = false + runOnUiThread { + uiManager.appendToUi("\n[Memory] 已更新用户画像: $activeUserId\n") + } + } catch (e: Exception) { + Log.w(TAG_LLM, "Profile parse failed: ${e.message}") + } + } + } + + private fun requestLocalProfileExtraction(dialogue: String, onResult: (String) -> Unit) { + try { + val local = llmManager + if (local == null) { + onResult("{}") + return + } + localThoughtSilentMode = true + pendingLocalProfileCallback = onResult + Log.i(TAG_LLM, "Routing profile extraction to LOCAL") + local.generateResponseWithSystem( + "你是信息抽取器。仅输出JSON对象，不要其他文字。字段为name,age,gender,hobbies,summary。", + "请从以下对话提取用户信息，未知填空字符串：\n$dialogue" + ) + } catch (e: Exception) { + pendingLocalProfileCallback = null + localThoughtSilentMode = false + Log.e(TAG_LLM, "requestLocalProfileExtraction failed: ${e.message}", e) + onResult("{}") + } + } + + private fun parseFirstJsonObject(text: String): JSONObject { + val raw = text.trim() + val start = raw.indexOf('{') + val end = raw.lastIndexOf('}') + if (start >= 0 && end > start) { + return JSONObject(raw.substring(start, end + 1)) + } + return JSONObject(raw) + } /** - * 初始化 LLM 管理器 + * 初始化本地 LLM（仅用于回忆状态） */ private fun initLLM() { try { - Log.i(TAG_LLM, "initLLM called, useLocalLLM=$useLocalLLM") + Log.i(TAG_LLM, "initLLM called for memory-local model") llmManager?.destroy() llmManager = null - if (useLocalLLM) { - // // 本地 LLM 初始化前，先暂停/释放重模块 - // Log.i(AppConfig.TAG, "Pausing camera and releasing face detection before LLM initialization") - // stopCameraPreviewAndDetection() - // try { - // faceDetectionPipeline.release() - // Log.i(AppConfig.TAG, "Face detection pipeline released") - // } catch (e: Exception) { - // Log.w(AppConfig.TAG, "Failed to release face detection pipeline: ${e.message}") - // } - - // // 释放 VAD 管理器 - // try { - // vadManager.release() - // Log.i(AppConfig.TAG, "VAD manager released") - // } catch (e: Exception) { - // Log.w(AppConfig.TAG, "Failed to release VAD manager: ${e.message}") - // } - - val modelPath = FileHelper.getLLMModelPath(applicationContext) - if (!File(modelPath).exists()) { - throw IllegalStateException("RKLLM model file missing: $modelPath") - } - Log.i(AppConfig.TAG, "Initializing LLM with model path: $modelPath") - val localLlmResponseBuffer = StringBuilder() - llmManager = LLMManager(modelPath, object : LLMManagerCallback { - override fun onThinking(msg: String, finished: Boolean) { - // 处理思考过程 - Log.d(TAG_LLM, "LOCAL onThinking finished=$finished msg=${msg.take(60)}") - runOnUiThread { - if (!finished && enableStreaming) { - uiManager.appendToUi("\n[LLM] 思考中: $msg\n") - } - } - } - - override fun onResult(msg: String, finished: Boolean) { - // 处理生成结果 - Log.d(TAG_LLM, "LOCAL onResult finished=$finished len=${msg.length}") - runOnUiThread { - if (!finished) { - localLlmResponseBuffer.append(msg) - if (enableStreaming) { - uiManager.appendToUi(msg) - } - } else { - val finalText = localLlmResponseBuffer.toString().trim() - localLlmResponseBuffer.setLength(0) - if (!enableStreaming && finalText.isNotEmpty()) { - uiManager.appendToUi("$finalText\n") - } - uiManager.appendToUi("\n\n[LLM] 生成完成\n") - llmInFlight = false - if (finalText.isNotEmpty()) { - ttsController.enqueueSegment(finalText) - ttsController.enqueueEnd() - } else { - Log.w(TAG_LLM, "LOCAL final text is empty, skip TTS enqueue") - } - } - } - } - }) - Log.i(AppConfig.TAG, "LLM initialized successfully") - Log.i(TAG_LLM, "LOCAL LLM initialized") - } else { - // 使用云端 LLM，不需要初始化本地 LLM - Log.i(AppConfig.TAG, "Using cloud LLM, skipping local LLM initialization") - Log.i(TAG_LLM, "CLOUD mode active") + val modelPath = FileHelper.getLLMModelPath(applicationContext) + if (!File(modelPath).exists()) { + throw IllegalStateException("RKLLM model file missing: $modelPath") } + Log.i(AppConfig.TAG, "Initializing local memory LLM with model path: $modelPath") + val localLlmResponseBuffer = StringBuilder() + llmManager = LLMManager(modelPath, object : LLMManagerCallback { + override fun onThinking(msg: String, finished: Boolean) { + Log.d(TAG_LLM, "LOCAL onThinking finished=$finished msg=${msg.take(60)}") + } + + override fun onResult(msg: String, finished: Boolean) { + Log.d(TAG_LLM, "LOCAL onResult finished=$finished len=${msg.length}") + runOnUiThread { + if (!finished) { + localLlmResponseBuffer.append(msg) + if (enableStreaming && !localThoughtSilentMode) { + uiManager.appendToUi(msg) + } + return@runOnUiThread + } + val finalText = localLlmResponseBuffer.toString().trim() + localLlmResponseBuffer.setLength(0) + val profileCallback = pendingLocalProfileCallback + pendingLocalProfileCallback = null + if (profileCallback != null) { + profileCallback(finalText) + localThoughtSilentMode = false + return@runOnUiThread + } + val callback = pendingLocalThoughtCallback + pendingLocalThoughtCallback = null + if (callback != null) { + callback(finalText) + localThoughtSilentMode = false + return@runOnUiThread + } + if (!localThoughtSilentMode && finalText.isNotEmpty()) { + uiManager.appendToUi("$finalText\n") + ttsController.enqueueSegment(finalText) + ttsController.enqueueEnd() + } + localThoughtSilentMode = false + } + } + }) + Log.i(TAG_LLM, "LOCAL memory LLM initialized") + useLocalLLM = true } catch (e: Exception) { Log.e(AppConfig.TAG, "Failed to initialize LLM: ${e.message}", e) Log.e(TAG_LLM, "LOCAL init failed: ${e.message}", e) @@ -849,35 +990,27 @@ class Live2DChatActivity : AppCompatActivity() { } /** - * 使用 LLM 生成回复 + * 回忆状态调用本地 LLM，仅用于 memory/what-are-you-thinking */ - private fun generateResponse(userInput: String) { + private fun requestLocalThought(prompt: String, onResult: (String) -> Unit) { try { - if (useLocalLLM) { - val systemPrompt = "你是一个友好的数字人助手，回答要简洁明了。" - Log.d(AppConfig.TAG, "Generating response for: $userInput") - val local = llmManager - if (local == null) { - Log.e(TAG_LLM, "LOCAL LLM manager is null, fallback to CLOUD") - cloudApiManager.callLLM(userInput) - return - } - Log.i(TAG_LLM, "LOCAL generateResponseWithSystem") - local.generateResponseWithSystem(systemPrompt, userInput) - } else { - // 使用云端 LLM - Log.d(AppConfig.TAG, "Using cloud LLM for response: $userInput") - Log.i(TAG_LLM, "CLOUD callLLM") - // 调用云端 LLM - cloudApiManager.callLLM(userInput) + val local = llmManager + if (local == null) { + onResult("我在想，下次见面可以聊聊今天的新鲜事。") + return } + localThoughtSilentMode = true + pendingLocalThoughtCallback = onResult + Log.i(TAG_LLM, "Routing memory thought to LOCAL") + local.generateResponseWithSystem( + "你是数字人内心独白模块，输出一句简短温和的想法。", + prompt + ) } catch (e: Exception) { - Log.e(AppConfig.TAG, "Failed to generate response: ${e.message}", e) - Log.e(TAG_LLM, "generateResponse failed: ${e.message}", e) - runOnUiThread { - uiManager.appendToUi("\n\n[Error] LLM 生成失败: ${e.message}\n") - llmInFlight = false - } + Log.e(TAG_LLM, "requestLocalThought failed: ${e.message}", e) + pendingLocalThoughtCallback = null + localThoughtSilentMode = false + onResult("我在想，下次见面可以聊聊今天的新鲜事。") } } } \ No newline at end of file diff --git a/app/src/main/java/com/digitalperson/cloud/CloudApiManager.java b/app/src/main/java/com/digitalperson/cloud/CloudApiManager.java index 104aef1..4bf6ba4 100644 --- a/app/src/main/java/com/digitalperson/cloud/CloudApiManager.java +++ b/app/src/main/java/com/digitalperson/cloud/CloudApiManager.java @@ -241,6 +241,13 @@ public class CloudApiManager { mConversationHistory = new JSONArray(); } + /** + * 添加助手消息到对话历史 + */ + public void addAssistantMessage(String content) { + addMessageToHistory("assistant", content); + } + public void callTTS(String text, File outputFile) { if (mListener != null) { mMainHandler.post(() -> { diff --git a/app/src/main/java/com/digitalperson/face/FaceDetectionPipeline.kt b/app/src/main/java/com/digitalperson/face/FaceDetectionPipeline.kt index 33c4207..686dd9e 100644 --- a/app/src/main/java/com/digitalperson/face/FaceDetectionPipeline.kt +++ b/app/src/main/java/com/digitalperson/face/FaceDetectionPipeline.kt @@ -31,7 +31,7 @@ data class FaceDetectionResult( class FaceDetectionPipeline( private val context: Context, private val onResult: (FaceDetectionResult) -> Unit, - private val onGreeting: (String) -> Unit, + private val onPresenceChanged: (present: Boolean, faceIdentityId: String?, recognizedName: String?) -> Unit, ) { private val engine = RetinaFaceEngineRKNN() private val recognizer = FaceRecognizer(context) @@ -41,8 +41,9 @@ class FaceDetectionPipeline( private var trackFace: FaceBox? = null private var trackId: Long = 0 private var trackStableSinceMs: Long = 0 - private var greetedTrackId: Long = -1 - private var lastGreetMs: Long = 0 + private var analyzedTrackId: Long = -1 + private var lastFaceIdentityId: String? = null + private var lastRecognizedName: String? = null fun initialize(): Boolean { val detectorOk = engine.initialize(context) @@ -98,8 +99,9 @@ class FaceDetectionPipeline( // ) // } - maybeRecognizeAndGreet(bitmap, filteredFaces) + maybeRecognize(bitmap, filteredFaces) withContext(Dispatchers.Main) { + onPresenceChanged(filteredFaces.isNotEmpty(), lastFaceIdentityId, lastRecognizedName) onResult(FaceDetectionResult(width, height, filteredFaces)) } } catch (t: Throwable) { @@ -111,11 +113,14 @@ class FaceDetectionPipeline( } } - private suspend fun maybeRecognizeAndGreet(bitmap: Bitmap, faces: List) { + private fun maybeRecognize(bitmap: Bitmap, faces: List) { val now = System.currentTimeMillis() if (faces.isEmpty()) { trackFace = null trackStableSinceMs = 0 + analyzedTrackId = -1 + lastFaceIdentityId = null + lastRecognizedName = null return } @@ -123,62 +128,30 @@ class FaceDetectionPipeline( val prev = trackFace if (prev == null || iou(prev, primary) < AppConfig.Face.TRACK_IOU_THRESHOLD) { trackId += 1 - greetedTrackId = -1 trackStableSinceMs = now + analyzedTrackId = -1 + lastFaceIdentityId = null + lastRecognizedName = null } trackFace = primary val stableMs = now - trackStableSinceMs val frontal = isFrontal(primary, bitmap.width, bitmap.height) - val coolingDown = (now - lastGreetMs) < AppConfig.FaceRecognition.GREETING_COOLDOWN_MS - if (stableMs < AppConfig.Face.STABLE_MS || !frontal || greetedTrackId == trackId || coolingDown) { + if (stableMs < AppConfig.Face.STABLE_MS || !frontal) { + return + } + if (analyzedTrackId == trackId) { return } - val match = recognizer.identify(bitmap, primary) - - Log.d(AppConfig.TAG, "[Face] Recognition result: matchedName=${match.matchedName}, similarity=${match.similarity}") - - // 检查是否需要保存新人脸 - if (match.matchedName.isNullOrBlank()) { - Log.d(AppConfig.TAG, "[Face] No match found, attempting to add new face") - // 提取人脸特征 - val embedding = extractEmbedding(bitmap, primary) - Log.d(AppConfig.TAG, "[Face] Extracted embedding size: ${embedding.size}") - if (embedding.isNotEmpty()) { - // 尝试添加新人脸 - val added = recognizer.addNewFace(embedding) - Log.d(AppConfig.TAG, "[Face] Add new face result: $added") - if (added) { - Log.i(AppConfig.TAG, "[Face] New face added to database") - } else { - Log.i(AppConfig.TAG, "[Face] Face already exists in database (similar face found)") - } - } else { - Log.w(AppConfig.TAG, "[Face] Failed to extract embedding") - } - } else { - Log.d(AppConfig.TAG, "[Face] Matched existing face: ${match.matchedName}") - } - - val greeting = if (!match.matchedName.isNullOrBlank()) { - "你好，${match.matchedName}！" - } else { - "你好，很高兴见到你。" - } - greetedTrackId = trackId - lastGreetMs = now + val match = recognizer.resolveIdentity(bitmap, primary) + analyzedTrackId = trackId + lastFaceIdentityId = match.matchedId?.let { "face_$it" } + lastRecognizedName = match.matchedName Log.i( AppConfig.TAG, - "[Face] greeting track=$trackId stable=${stableMs}ms frontal=$frontal matched=${match.matchedName} score=${match.similarity}" + "[Face] stable track=$trackId faceId=${lastFaceIdentityId} matched=${match.matchedName} score=${match.similarity}" ) - withContext(Dispatchers.Main) { - onGreeting(greeting) - } - } - - private fun extractEmbedding(bitmap: Bitmap, face: FaceBox): FloatArray { - return recognizer.extractEmbedding(bitmap, face) } private fun isFrontal(face: FaceBox, frameW: Int, frameH: Int): Boolean { diff --git a/app/src/main/java/com/digitalperson/face/FaceFeatureStore.kt b/app/src/main/java/com/digitalperson/face/FaceFeatureStore.kt index 52cb025..5ab807e 100644 --- a/app/src/main/java/com/digitalperson/face/FaceFeatureStore.kt +++ b/app/src/main/java/com/digitalperson/face/FaceFeatureStore.kt @@ -11,7 +11,7 @@ import java.nio.ByteOrder data class FaceProfile( val id: Long, - val name: String, + val name: String?, val embedding: FloatArray, ) @@ -21,7 +21,7 @@ class FaceFeatureStore(context: Context) : SQLiteOpenHelper(context, DB_NAME, nu """ CREATE TABLE IF NOT EXISTS face_profiles ( id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL UNIQUE, + name TEXT, embedding BLOB NOT NULL, updated_at INTEGER NOT NULL ) @@ -55,9 +55,8 @@ class FaceFeatureStore(context: Context) : SQLiteOpenHelper(context, DB_NAME, nu return list } - fun upsertProfile(name: String, embedding: FloatArray) { - // 确保名字不为null，使用空字符串作为默认值 - val safeName = name.takeIf { it.isNotBlank() } ?: "" + fun insertProfile(name: String?, embedding: FloatArray): Long { + val safeName = name?.takeIf { it.isNotBlank() } val values = ContentValues().apply { put("name", safeName) put("embedding", floatArrayToBlob(embedding)) @@ -67,9 +66,10 @@ class FaceFeatureStore(context: Context) : SQLiteOpenHelper(context, DB_NAME, nu "face_profiles", null, values, - SQLiteDatabase.CONFLICT_REPLACE + SQLiteDatabase.CONFLICT_NONE ) - Log.i(AppConfig.TAG, "[FaceFeatureStore] upsertProfile name='$safeName' rowId=$rowId dim=${embedding.size}") + Log.i(AppConfig.TAG, "[FaceFeatureStore] insertProfile name='$safeName' rowId=$rowId dim=${embedding.size}") + return rowId } private fun floatArrayToBlob(values: FloatArray): ByteArray { @@ -88,6 +88,6 @@ class FaceFeatureStore(context: Context) : SQLiteOpenHelper(context, DB_NAME, nu companion object { private const val DB_NAME = "face_feature.db" - private const val DB_VERSION = 1 + private const val DB_VERSION = 2 } } diff --git a/app/src/main/java/com/digitalperson/face/FaceRecognizer.kt b/app/src/main/java/com/digitalperson/face/FaceRecognizer.kt index 9ccbd91..85a68b5 100644 --- a/app/src/main/java/com/digitalperson/face/FaceRecognizer.kt +++ b/app/src/main/java/com/digitalperson/face/FaceRecognizer.kt @@ -8,6 +8,7 @@ import com.digitalperson.engine.ArcFaceEngineRKNN import kotlin.math.sqrt data class FaceRecognitionResult( + val matchedId: Long?, val matchedName: String?, val similarity: Float, val embeddingDim: Int, @@ -40,10 +41,11 @@ class FaceRecognizer(context: Context) { } fun identify(bitmap: Bitmap, face: FaceBox): FaceRecognitionResult { - if (!initialized) return FaceRecognitionResult(null, 0f, 0) + if (!initialized) return FaceRecognitionResult(null, null, 0f, 0) val embedding = extractEmbedding(bitmap, face) - if (embedding.isEmpty()) return FaceRecognitionResult(null, 0f, 0) + if (embedding.isEmpty()) return FaceRecognitionResult(null, null, 0f, 0) + var bestId: Long? = null var bestName: String? = null var bestScore = -1f for (p in cache) { @@ -51,13 +53,14 @@ class FaceRecognizer(context: Context) { val score = cosineSimilarity(embedding, p.embedding) if (score > bestScore) { bestScore = score + bestId = p.id bestName = p.name } } if (bestScore >= AppConfig.FaceRecognition.SIMILARITY_THRESHOLD) { - return FaceRecognitionResult(bestName, bestScore, embedding.size) + return FaceRecognitionResult(bestId, bestName, bestScore, embedding.size) } - return FaceRecognitionResult(null, bestScore, embedding.size) + return FaceRecognitionResult(null, null, bestScore, embedding.size) } fun extractEmbedding(bitmap: Bitmap, face: FaceBox): FloatArray { @@ -65,38 +68,28 @@ class FaceRecognizer(context: Context) { return engine.extractEmbedding(bitmap, face.left, face.top, face.right, face.bottom) } - fun addOrUpdateProfile(name: String?, embedding: FloatArray) { + private fun addProfile(name: String?, embedding: FloatArray): Long { val normalized = normalize(embedding) - store.upsertProfile(name ?: "", normalized) - // 移除旧的记录（如果存在） - if (name != null) { - cache.removeAll { it.name == name } + val rowId = store.insertProfile(name, normalized) + if (rowId > 0) { + cache.add(FaceProfile(id = rowId, name = name, embedding = normalized)) } - cache.add(FaceProfile(id = -1L, name = name ?: "", embedding = normalized)) + return rowId } - fun addNewFace(embedding: FloatArray): Boolean { - Log.d(AppConfig.TAG, "[FaceRecognizer] addNewFace: embedding size=${embedding.size}, cache size=${cache.size}") - - // 检查是否已经存在相似的人脸 - for (p in cache) { - if (p.embedding.size != embedding.size) { - Log.d(AppConfig.TAG, "[FaceRecognizer] Skipping profile with different embedding size: ${p.embedding.size}") - continue - } - val score = cosineSimilarity(embedding, p.embedding) - Log.d(AppConfig.TAG, "[FaceRecognizer] Comparing with profile '${p.name}': similarity=$score, threshold=${AppConfig.FaceRecognition.SIMILARITY_THRESHOLD}") - if (score >= AppConfig.FaceRecognition.SIMILARITY_THRESHOLD) { - // 已经存在相似的人脸，不需要添加 - Log.i(AppConfig.TAG, "[FaceRecognizer] Similar face found: ${p.name} with similarity=$score, not adding new face") - return false - } - } - - // 添加新人脸，名字为null - Log.i(AppConfig.TAG, "[FaceRecognizer] No similar face found, adding new face") - addOrUpdateProfile(null, embedding) - return true + fun resolveIdentity(bitmap: Bitmap, face: FaceBox): FaceRecognitionResult { + val match = identify(bitmap, face) + if (match.matchedId != null) return match + val embedding = extractEmbedding(bitmap, face) + if (embedding.isEmpty()) return match + val newId = addProfile(name = null, embedding = embedding) + if (newId <= 0L) return match + return FaceRecognitionResult( + matchedId = newId, + matchedName = null, + similarity = match.similarity, + embeddingDim = embedding.size + ) } fun release() { diff --git a/app/src/main/java/com/digitalperson/interaction/DigitalHumanInteractionController.kt b/app/src/main/java/com/digitalperson/interaction/DigitalHumanInteractionController.kt new file mode 100644 index 0000000..4297d6c --- /dev/null +++ b/app/src/main/java/com/digitalperson/interaction/DigitalHumanInteractionController.kt @@ -0,0 +1,268 @@ +package com.digitalperson.interaction + +import android.util.Log +import com.digitalperson.config.AppConfig +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch + +enum class InteractionState { + IDLE, + MEMORY, + GREETING, + WAITING_REPLY, + DIALOGUE, + PROACTIVE, + FAREWELL, +} + +enum class LlmRoute { + CLOUD, + LOCAL, +} + +interface InteractionActionHandler { + fun onStateChanged(state: InteractionState) + fun playMotion(motionName: String) + fun appendText(text: String) + fun speak(text: String) + fun requestCloudReply(userText: String) + fun requestLocalThought(prompt: String, onResult: (String) -> Unit) + fun onRememberUser(faceIdentityId: String, name: String?) + fun saveThought(thought: String) + fun loadLatestThought(): String? + fun addToChatHistory(role: String, content: String) + fun addAssistantMessageToCloudHistory(content: String) +} + +class DigitalHumanInteractionController( + private val scope: CoroutineScope, + private val handler: InteractionActionHandler, +) { + private val TAG: String = "DigitalHumanInteraction" + private var state: InteractionState = InteractionState.IDLE + private var facePresent: Boolean = false + private var recognizedName: String? = null + private var proactiveRound = 0 + private var hasPendingUserReply = false + + private var faceStableJob: Job? = null + private var waitReplyJob: Job? = null + private var proactiveJob: Job? = null + private var memoryJob: Job? = null + private var farewellJob: Job? = null + + fun start() { + transitionTo(InteractionState.IDLE) + scheduleMemoryMode() + } + +fun onFacePresenceChanged(present: Boolean, faceIdentityId: String?, recognized: String?) { + Log.d(TAG, "onFacePresenceChanged: present=$present, faceIdentityId=$faceIdentityId, recognized=$recognized, state=$state") + + facePresent = present + if (!faceIdentityId.isNullOrBlank()) { + handler.onRememberUser(faceIdentityId, recognized) + } + if (!recognized.isNullOrBlank()) { + recognizedName = recognized + } + + // 统一延迟处理 + faceStableJob?.cancel() + faceStableJob = scope.launch { + delay(AppConfig.Face.STABLE_MS) + + if (present) { + // 人脸出现后的处理 + if (facePresent && (state == InteractionState.IDLE || state == InteractionState.MEMORY)) { + enterGreeting() + } else if (state == InteractionState.FAREWELL) { + enterGreeting() + } + } else { + // 人脸消失后的处理 + if (state != InteractionState.IDLE && state != InteractionState.MEMORY && state != InteractionState.FAREWELL) { + enterFarewell() + } else { + scheduleMemoryMode() + } + } + } +} + + fun onUserAsrText(text: String) { + val userText = text.trim() + if (userText.isBlank()) return + + if (userText.contains("你在想什么")) { + val thought = handler.loadLatestThought() + if (!thought.isNullOrBlank()) { + handler.speak("我刚才在想：$thought") + handler.appendText("\n[回忆] $thought\n") + transitionTo(InteractionState.DIALOGUE) + handler.playMotion("haru_g_m15.motion3.json") + return + } + } + + if (userText.contains("再见")) { + enterFarewell() + return + } + + hasPendingUserReply = true + when (state) { + InteractionState.WAITING_REPLY, InteractionState.PROACTIVE, InteractionState.GREETING, InteractionState.DIALOGUE -> { + transitionTo(InteractionState.DIALOGUE) + requestDialogueReply(userText) + } + InteractionState.MEMORY, InteractionState.IDLE -> { + transitionTo(InteractionState.DIALOGUE) + requestDialogueReply(userText) + } + InteractionState.FAREWELL -> { + enterGreeting() + } + } + } + + fun onDialogueResponseFinished() { + if (!facePresent) { + enterFarewell() + return + } + transitionTo(InteractionState.WAITING_REPLY) + handler.playMotion("haru_g_m17.motion3.json") + scheduleWaitingReplyTimeout() + } + + private fun enterGreeting() { + transitionTo(InteractionState.GREETING) + val greet = if (!recognizedName.isNullOrBlank()) { + handler.playMotion("haru_g_m22.motion3.json") + "你好，$recognizedName，很高兴再次见到你。" + } else { + handler.playMotion("haru_g_m01.motion3.json") + "你好，很高兴见到你。" + } + handler.speak(greet) + handler.appendText("\n[问候] $greet\n") + transitionTo(InteractionState.WAITING_REPLY) + handler.playMotion("haru_g_m17.motion3.json") + scheduleWaitingReplyTimeout() + } + + private fun scheduleWaitingReplyTimeout() { + waitReplyJob?.cancel() + waitReplyJob = scope.launch { + hasPendingUserReply = false + delay(10_000) + if (state != InteractionState.WAITING_REPLY || hasPendingUserReply) return@launch + if (facePresent) { + enterProactive() + } else { + enterFarewell() + } + } + } + + private fun enterProactive() { + transitionTo(InteractionState.PROACTIVE) + proactiveRound = 0 + askProactiveTopic() + } + + private fun askProactiveTopic() { + proactiveJob?.cancel() + val topics = listOf( + "我出一道数学题考考你吧？1+6等于多少？", + "我们上完厕所应该干什么呀？", + "你喜欢什么颜色呀？", + ) + val idx = proactiveRound.coerceIn(0, topics.lastIndex) + val topic = topics[idx] + handler.playMotion(if (proactiveRound == 0) "haru_g_m15.motion3.json" else "haru_g_m22.motion3.json") + handler.speak(topic) + handler.appendText("\n[主动引导] $topic\n") + // 将引导内容添加到对话历史中 + handler.addToChatHistory("助手", topic) + // 将引导内容添加到云对话历史中 + handler.addAssistantMessageToCloudHistory(topic) + + proactiveJob = scope.launch { + hasPendingUserReply = false + delay(20_000) + if (state != InteractionState.PROACTIVE || hasPendingUserReply) return@launch + if (!facePresent) { + enterFarewell() + return@launch + } + proactiveRound += 1 + if (proactiveRound < 3) { + askProactiveTopic() + } else { + transitionTo(InteractionState.WAITING_REPLY) + handler.playMotion("haru_g_m17.motion3.json") + scheduleWaitingReplyTimeout() + } + } + } + + private fun enterFarewell() { + transitionTo(InteractionState.FAREWELL) + handler.playMotion("haru_g_idle.motion3.json") + handler.speak("再见咯") + farewellJob?.cancel() + farewellJob = scope.launch { + delay(3_000) + transitionTo(InteractionState.IDLE) + handler.playMotion("haru_g_idle.motion3.json") + scheduleMemoryMode() + } + } + + private fun scheduleMemoryMode() { + memoryJob?.cancel() + if (facePresent) return + memoryJob = scope.launch { + delay(30_000) + if (facePresent || state != InteractionState.IDLE) return@launch + transitionTo(InteractionState.MEMORY) + handler.playMotion("haru_g_m15.motion3.json") + val prompt = "请基于最近互动写一句简短内心想法，口吻温和自然。" + handler.requestLocalThought(prompt) { thought -> + val finalThought = thought.trim().ifBlank { "我在想，下次见面要聊点有趣的新话题。" } + handler.saveThought(finalThought) + handler.appendText("\n[回忆] $finalThought\n") + if (!facePresent && state == InteractionState.MEMORY) { + transitionTo(InteractionState.IDLE) + handler.playMotion("haru_g_idle.motion3.json") + scheduleMemoryMode() + } + } + } + } + + private fun requestDialogueReply(userText: String) { + waitReplyJob?.cancel() + proactiveJob?.cancel() + // 按产品要求：用户对话统一走云端LLM + handler.requestCloudReply(userText) + } + + private fun transitionTo(newState: InteractionState) { + if (state == newState) return + state = newState + handler.onStateChanged(newState) + } + + fun stop() { + faceStableJob?.cancel() + waitReplyJob?.cancel() + proactiveJob?.cancel() + memoryJob?.cancel() + farewellJob?.cancel() + } +} diff --git a/app/src/main/java/com/digitalperson/interaction/UserMemoryStore.kt b/app/src/main/java/com/digitalperson/interaction/UserMemoryStore.kt new file mode 100644 index 0000000..e07d972 --- /dev/null +++ b/app/src/main/java/com/digitalperson/interaction/UserMemoryStore.kt @@ -0,0 +1,177 @@ +package com.digitalperson.interaction + +import android.content.ContentValues +import android.content.Context +import android.database.sqlite.SQLiteDatabase +import android.database.sqlite.SQLiteOpenHelper + +data class UserMemory( + val userId: String, + val displayName: String?, + val lastSeenAt: Long, + val age: String?, + val gender: String?, + val hobbies: String?, + val preferences: String?, + val lastTopics: String?, + val lastThought: String?, + val profileSummary: String?, +) + +class UserMemoryStore(context: Context) : SQLiteOpenHelper(context, DB_NAME, null, DB_VERSION) { + private val memoryCache = LinkedHashMap() + @Volatile private var latestThoughtCache: String? = null + + override fun onCreate(db: SQLiteDatabase) { + db.execSQL( + """ + CREATE TABLE IF NOT EXISTS user_memory ( + user_id TEXT PRIMARY KEY, + display_name TEXT, + last_seen_at INTEGER NOT NULL, + age TEXT, + gender TEXT, + hobbies TEXT, + preferences TEXT, + last_topics TEXT, + last_thought TEXT, + profile_summary TEXT + ) + """.trimIndent() + ) + } + + override fun onUpgrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { + db.execSQL("DROP TABLE IF EXISTS user_memory") + onCreate(db) + } + + fun upsertUserSeen(userId: String, displayName: String?) { + val existing = memoryCache[userId] ?: getMemory(userId) + val now = System.currentTimeMillis() + val mergedName = displayName?.takeIf { it.isNotBlank() } ?: existing?.displayName + val values = ContentValues().apply { + put("user_id", userId) + put("display_name", mergedName) + put("last_seen_at", now) + put("age", existing?.age) + put("gender", existing?.gender) + put("hobbies", existing?.hobbies) + put("preferences", existing?.preferences) + put("last_topics", existing?.lastTopics) + put("last_thought", existing?.lastThought) + put("profile_summary", existing?.profileSummary) + } + writableDatabase.insertWithOnConflict("user_memory", null, values, SQLiteDatabase.CONFLICT_REPLACE) + memoryCache[userId] = UserMemory( + userId = userId, + displayName = mergedName, + lastSeenAt = now, + age = existing?.age, + gender = existing?.gender, + hobbies = existing?.hobbies, + preferences = existing?.preferences, + lastTopics = existing?.lastTopics, + lastThought = existing?.lastThought, + profileSummary = existing?.profileSummary, + ) + } + + fun updateDisplayName(userId: String, displayName: String?) { + if (displayName.isNullOrBlank()) return + upsertUserSeen(userId, displayName) + } + + fun updateThought(userId: String, thought: String) { + upsertUserSeen(userId, null) + val now = System.currentTimeMillis() + val values = ContentValues().apply { + put("last_thought", thought) + put("last_seen_at", now) + } + writableDatabase.update("user_memory", values, "user_id=?", arrayOf(userId)) + latestThoughtCache = thought + val cached = memoryCache[userId] + memoryCache[userId] = UserMemory( + userId = userId, + displayName = cached?.displayName, + lastSeenAt = now, + age = cached?.age, + gender = cached?.gender, + hobbies = cached?.hobbies, + preferences = cached?.preferences, + lastTopics = cached?.lastTopics, + lastThought = thought, + profileSummary = cached?.profileSummary, + ) + } + + fun updateProfile(userId: String, age: String?, gender: String?, hobbies: String?, summary: String?) { + upsertUserSeen(userId, null) + val now = System.currentTimeMillis() + val values = ContentValues().apply { + if (age != null) put("age", age) + if (gender != null) put("gender", gender) + if (hobbies != null) put("hobbies", hobbies) + if (summary != null) put("profile_summary", summary) + put("last_seen_at", now) + } + writableDatabase.update("user_memory", values, "user_id=?", arrayOf(userId)) + val cached = memoryCache[userId] + memoryCache[userId] = UserMemory( + userId = userId, + displayName = cached?.displayName, + lastSeenAt = now, + age = age ?: cached?.age, + gender = gender ?: cached?.gender, + hobbies = hobbies ?: cached?.hobbies, + preferences = cached?.preferences, + lastTopics = cached?.lastTopics, + lastThought = cached?.lastThought, + profileSummary = summary ?: cached?.profileSummary, + ) + } + + fun getMemory(userId: String): UserMemory? { + memoryCache[userId]?.let { return it } + readableDatabase.rawQuery( + "SELECT user_id, display_name, last_seen_at, age, gender, hobbies, preferences, last_topics, last_thought, profile_summary FROM user_memory WHERE user_id=?", + arrayOf(userId) + ).use { c -> + if (!c.moveToFirst()) return null + val memory = UserMemory( + userId = c.getString(0), + displayName = c.getString(1), + lastSeenAt = c.getLong(2), + age = c.getString(3), + gender = c.getString(4), + hobbies = c.getString(5), + preferences = c.getString(6), + lastTopics = c.getString(7), + lastThought = c.getString(8), + profileSummary = c.getString(9), + ) + memoryCache[userId] = memory + if (!memory.lastThought.isNullOrBlank()) { + latestThoughtCache = memory.lastThought + } + return memory + } + } + + fun getLatestThought(): String? { + latestThoughtCache?.let { return it } + readableDatabase.rawQuery( + "SELECT last_thought FROM user_memory WHERE last_thought IS NOT NULL AND last_thought != '' ORDER BY last_seen_at DESC LIMIT 1", + null + ).use { c -> + if (!c.moveToFirst()) return null + return c.getString(0).also { latestThoughtCache = it } + } + } + + companion object { + private const val DB_NAME = "digital_human_memory.db" + private const val DB_VERSION = 2 + } +} diff --git a/app/src/main/java/com/digitalperson/ui/Live2DUiManager.kt b/app/src/main/java/com/digitalperson/ui/Live2DUiManager.kt index f390391..cef9e1b 100644 --- a/app/src/main/java/com/digitalperson/ui/Live2DUiManager.kt +++ b/app/src/main/java/com/digitalperson/ui/Live2DUiManager.kt @@ -98,19 +98,25 @@ class Live2DUiManager(private val activity: Activity) { } fun appendToUi(s: String) { - lastUiText += s - textView?.text = lastUiText - scrollView?.post { scrollView?.fullScroll(ScrollView.FOCUS_DOWN) } + activity.runOnUiThread { + lastUiText += s + textView?.text = lastUiText + scrollView?.post { scrollView?.fullScroll(ScrollView.FOCUS_DOWN) } + } } fun clearText() { - lastUiText = "" - textView?.text = "" + activity.runOnUiThread { + lastUiText = "" + textView?.text = "" + } } fun setText(text: String) { - lastUiText = text - textView?.text = text + activity.runOnUiThread { + lastUiText = text + textView?.text = text + } } fun setButtonsEnabled(startEnabled: Boolean = false, stopEnabled: Boolean = false, recordEnabled: Boolean = true) {