3dtours/photobooth-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize: downscale viewfinder preview to prevent camera lag, remove unused MiDaS depth model and TensorFlow Lite dependencies

Browse Source
This commit is contained in:
Lộc Phạm
2026-07-06 18:32:40 +07:00
parent 06614f92e6
commit fe095304cc
3 changed files with 105 additions and 302 deletions
Download Patch File Download Diff File Expand all files Collapse all files
android-project/app/build.gradle.kts
-4
View File
@@ -63,10 +63,6 @@ dependencies {
// --- TensorFlow Lite for MiDaS ---
implementation("org.tensorflow:tensorflow-lite:2.12.0")
implementation("org.tensorflow:tensorflow-lite-support:0.4.4")
// --- Google MediaPipe Tasks Vision ---
implementation("com.google.mediapipe:tasks-vision:0.10.14")
}
android-project/app/src/main/java/com/photobooth/app/MainActivity.kt
+42 -235
View File
@@ -103,98 +103,6 @@ class MainActivity : AppCompatActivity() {
}
}
private var tfliteInterpreter: org.tensorflow.lite.Interpreter? = null
private fun getTfliteInterpreter(): org.tensorflow.lite.Interpreter? {
synchronized(aiInferenceLock) {
if (!isAiSupported) return null
if (tfliteInterpreter == null) {
try {
val assetFileDescriptor = assets.openFd("models/midas.tflite")
val inputStream = java.io.FileInputStream(assetFileDescriptor.fileDescriptor)
val fileChannel = inputStream.channel
val startOffset = assetFileDescriptor.startOffset
val declaredLength = assetFileDescriptor.declaredLength
val modelBuffer = fileChannel.map(java.nio.channels.FileChannel.MapMode.READ_ONLY, startOffset, declaredLength)
val options = org.tensorflow.lite.Interpreter.Options()
options.setNumThreads(4)
tfliteInterpreter = org.tensorflow.lite.Interpreter(modelBuffer, options)
} catch (e: UnsatisfiedLinkError) {
isAiSupported = false
runOnUiThread {
Toast.makeText(this, "Không hỗ trợ xử lý Depth trên thiết bị/emulator này (thiếu thư viện Native JNI)", Toast.LENGTH_LONG).show()
}
return null
} catch (e: Exception) {
isAiSupported = false
runOnUiThread {
Toast.makeText(this, "Lỗi khởi tạo Depth Interpreter: ${e.message}", Toast.LENGTH_LONG).show()
}
return null
}
}
return tfliteInterpreter
}
}
private fun runDepthInference(bitmap: android.graphics.Bitmap): FloatArray {
val interpreter = getTfliteInterpreter() ?: return FloatArray(256 * 256)
// Input: 256x256 RGB
val scaled = android.graphics.Bitmap.createScaledBitmap(bitmap, 256, 256, true)
val inputBuffer = java.nio.ByteBuffer.allocateDirect(1 * 256 * 256 * 3 * 4)
inputBuffer.order(java.nio.ByteOrder.nativeOrder())
val pixels = IntArray(256 * 256)
scaled.getPixels(pixels, 0, 256, 0, 0, 256, 256)
scaled.recycle()
val mean = floatArrayOf(0.485f, 0.456f, 0.406f)
val std = floatArrayOf(0.229f, 0.224f, 0.225f)
for (pixel in pixels) {
val r = ((pixel shr 16) and 0xFF) / 255.0f
val g = ((pixel shr 8) and 0xFF) / 255.0f
val b = (pixel and 0xFF) / 255.0f
inputBuffer.putFloat((r - mean[0]) / std[0])
inputBuffer.putFloat((g - mean[1]) / std[1])
inputBuffer.putFloat((b - mean[2]) / std[2])
}
inputBuffer.rewind()
val outputBuffer = java.nio.ByteBuffer.allocateDirect(1 * 256 * 256 * 4)
outputBuffer.order(java.nio.ByteOrder.nativeOrder())
synchronized(aiInferenceLock) {
interpreter.run(inputBuffer, outputBuffer)
}
outputBuffer.rewind()
val depthMap = FloatArray(256 * 256)
outputBuffer.asFloatBuffer().get(depthMap)
var minVal = Float.MAX_VALUE
var maxVal = Float.MIN_VALUE
for (v in depthMap) {
if (v < minVal) minVal = v
if (v > maxVal) maxVal = v
}
val range = maxVal - minVal
if (range > 0f) {
for (i in depthMap.indices) {
val normalized = (depthMap[i] - minVal) / range
depthMap[i] = 1.0f - normalized
}
} else {
java.util.Arrays.fill(depthMap, 0f)
}
return depthMap
}
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
@@ -1728,10 +1636,20 @@ class MainActivity : AppCompatActivity() {
matrix.postScale(-1f, 1f)
}
val rotated = android.graphics.Bitmap.createBitmap(src, 0, 0, src.width, src.height, matrix, true)
val ai = processAdvancedBackgroundAi(rotated, isStreamMode = true)
if (ai != rotated) {
// Tối ưu hóa giật lag: Downscale ảnh preview xuống 2 lần để giảm tải CPU xử lý vòng lặp pixel
val scaleFactor = 2
val downscaledW = (rotated.width / scaleFactor).coerceAtLeast(1)
val downscaledH = (rotated.height / scaleFactor).coerceAtLeast(1)
val downscaled = android.graphics.Bitmap.createScaledBitmap(rotated, downscaledW, downscaledH, true)
if (downscaled != rotated) {
rotated.recycle()
}
val ai = processAdvancedBackgroundAi(downscaled, isStreamMode = true)
if (ai != downscaled) {
downscaled.recycle()
}
return ai
}
@@ -1868,80 +1786,37 @@ class MainActivity : AppCompatActivity() {
try {
val segmenter = getImageSegmenter()
val interpreter = getTfliteInterpreter()
if (segmenter == null || interpreter == null) {
if (blurIntensity > 0f) {
val radius = (25 * blurIntensity).toInt().coerceIn(1, 25)
return blurBitmap(capturedImage, radius)
}
return capturedImage
}
// 1. Run MediaPipe Multiclass Image Segmentation to locate subject
if (segmenter != null) {
val mpImage = com.google.mediapipe.framework.image.BitmapImageBuilder(capturedImage).build()
val segmentationResult = synchronized(aiInferenceLock) {
segmenter.segment(mpImage)
}
val categoryMaskOptional = segmentationResult.categoryMask()
if (!categoryMaskOptional.isPresent) {
return capturedImage
}
if (categoryMaskOptional.isPresent) {
val categoryMask = categoryMaskOptional.get()
val categoryMaskBuffer: java.nio.ByteBuffer = ByteBufferExtractor.extract(categoryMask)
val categoryMaskBuffer = ByteBufferExtractor.extract(categoryMask)
categoryMaskBuffer.rewind()
val w = categoryMask.width
val h = categoryMask.height
val bufferCapacity = categoryMaskBuffer.capacity()
// --- THUẬT TOÁN 1: MASK SMOOTHING (LÀM MƯỢT BIÊN TOÀN DIỆN) ---
// Create and blur alpha mask
val alphaMask = FloatArray(w * h)
for (i in 0 until (w * h)) {
val classId = if (i < bufferCapacity) categoryMaskBuffer.get(i).toInt() else 0
alphaMask[i] = if (classId in 1..5) 1.0f else 0.0f
}
// Làm mượt mặt nạ để khử răng cưa và vết cắt cứng ở viền tóc/tai/vai
blurAlphaMaskBox(alphaMask, w, h, radius = 4)
// 2. Perform Monocular Depth Estimation via TFLite MiDaS model
val depthMap = runDepthInference(capturedImage)
// --- BƯỚC 1: TÍNH TIÊU CỰ ĐỐI TỰ ĐỘNG BẰNG CẢ 2 AI (DYNAMIC AUTO-FOCUS) ---
var totalDepthSum = 0f
var validPixelCount = 0
// Quét nhanh qua ma trận để tìm chiều sâu thực tế của riêng cơ thể người (classId in 1..5)
for (y in 0 until h step 8) {
for (x in 0 until w step 8) {
val idx = y * w + x
if (idx < bufferCapacity) {
val classId = categoryMaskBuffer.get(idx).toInt()
if (classId in 1..5) {
val maskX = (x * 256 / w).coerceIn(0, 255)
val maskY = (y * 256 / h).coerceIn(0, 255)
totalDepthSum += depthMap[maskY * 256 + maskX]
validPixelCount++
}
}
}
}
// Mốc tiêu cự chuẩn của đối tượng (Mặc định tâm 0.25f)
val objectFocalPoint = if (validPixelCount > 0) totalDepthSum / validPixelCount else 0.25f
val fieldTolerance = 0.12f // Độ dày trường ảnh để giữ nét trọn vẹn tai, cổ, vai
// 3. Prepare background image (supporting parallel replace + blur)
val bgBitmap = android.graphics.Bitmap.createBitmap(w, h, android.graphics.Bitmap.Config.ARGB_8888)
blurAlphaMaskBox(alphaMask, w, h, radius = 5)
// Prepare background (camera preview or custom background asset)
val baseBg = if (selectedBgAssetPath != null) {
try {
val bgInputStream = assets.open(selectedBgAssetPath!!)
val decodedBg = android.graphics.BitmapFactory.decodeStream(bgInputStream)
if (decodedBg != null) {
val tempScaled = android.graphics.Bitmap.createScaledBitmap(decodedBg, w, h, true)
if (tempScaled != decodedBg) {
decodedBg.recycle()
}
if (tempScaled != decodedBg) decodedBg.recycle()
tempScaled
} else {
val solidBg = android.graphics.Bitmap.createBitmap(w, h, android.graphics.Bitmap.Config.ARGB_8888)
@@ -1957,42 +1832,13 @@ class MainActivity : AppCompatActivity() {
capturedImage
}
// Build the background pixel matrices
val bgPixels = IntArray(w * h)
var blurStage1: android.graphics.Bitmap? = null
var blurStage2: android.graphics.Bitmap? = null
var blurStage3: android.graphics.Bitmap? = null
val bg1Pixels = if (blurIntensity > 0f) {
val r1 = (5 * blurIntensity).toInt().coerceIn(1, 25)
val b1 = blurBitmap(baseBg, r1)
blurStage1 = b1
val px = IntArray(w * h)
b1.getPixels(px, 0, w, 0, 0, w, h)
px
} else null
val bg2Pixels = if (blurIntensity > 0f) {
val r2 = (13 * blurIntensity).toInt().coerceIn(1, 25)
val b2 = blurBitmap(baseBg, r2)
blurStage2 = b2
val px = IntArray(w * h)
b2.getPixels(px, 0, w, 0, 0, w, h)
px
} else null
val bg3Pixels = if (blurIntensity > 0f) {
val r3 = (25 * blurIntensity).toInt().coerceIn(1, 25)
val b3 = blurBitmap(baseBg, r3)
blurStage3 = b3
val px = IntArray(w * h)
b3.getPixels(px, 0, w, 0, 0, w, h)
px
} else null
if (blurIntensity == 0f) {
// Just use baseBg directly
var blurredBg: android.graphics.Bitmap? = null
if (blurIntensity > 0f) {
val radius = (25 * blurIntensity).toInt().coerceIn(1, 25)
blurredBg = blurBitmap(baseBg, radius)
blurredBg.getPixels(bgPixels, 0, w, 0, 0, w, h)
} else {
baseBg.getPixels(bgPixels, 0, w, 0, 0, w, h)
}
@@ -2000,65 +1846,24 @@ class MainActivity : AppCompatActivity() {
baseBg.recycle()
}
// 4. Pixel loop processing: feathering, decontamination, alpha blending
val fgPixels = IntArray(w * h)
val outputPixels = IntArray(w * h)
capturedImage.getPixels(fgPixels, 0, w, 0, 0, w, h)
for (y in 0 until h) {
for (x in 0 until w) {
val idx = y * w + x
val maskX = (x * 256 / w).coerceIn(0, 255)
val maskY = (y * 256 / h).coerceIn(0, 255)
val distance = depthMap[maskY * 256 + maskX]
val relativeDistance = distance - objectFocalPoint
// --- THUẬT TOÁN 1: FEATHERING THEO CHIỀU SÂU & MEDIAPIPE ---
val focusAlpha = when {
relativeDistance <= 0f -> 1.0f
relativeDistance < fieldTolerance - 0.05f -> 1.0f
relativeDistance >= fieldTolerance + 0.05f -> 0.0f
else -> {
val t = (relativeDistance - (fieldTolerance - 0.05f)) / 0.10f
1.0f - cubicSmoothStep(0f, 1f, t)
}
}
val maskAlpha = alphaMask[idx]
val alpha = kotlin.math.max(maskAlpha, focusAlpha)
for (idx in 0 until (w * h)) {
val alpha = alphaMask[idx]
val fgPixel = fgPixels[idx]
var fgR = android.graphics.Color.red(fgPixel)
var fgG = android.graphics.Color.green(fgPixel)
var fgB = android.graphics.Color.blue(fgPixel)
val bgR: Int
val bgG: Int
val bgB: Int
if (blurIntensity > 0f && bg1Pixels != null && bg2Pixels != null && bg3Pixels != null) {
// Nền mờ dần theo khoảng cách vật lý thực tế từ MiDaS
val bgPixel = if (relativeDistance < 0.25f) {
bg1Pixels[idx]
} else if (relativeDistance >= 0.25f && relativeDistance < 0.5f) {
bg2Pixels[idx]
} else {
bg3Pixels[idx]
}
bgR = android.graphics.Color.red(bgPixel)
bgG = android.graphics.Color.green(bgPixel)
bgB = android.graphics.Color.blue(bgPixel)
} else {
val bgPixel = bgPixels[idx]
bgR = android.graphics.Color.red(bgPixel)
bgG = android.graphics.Color.green(bgPixel)
bgB = android.graphics.Color.blue(bgPixel)
}
val bgR = android.graphics.Color.red(bgPixel)
val bgG = android.graphics.Color.green(bgPixel)
val bgB = android.graphics.Color.blue(bgPixel)
// --- THUẬT TOÁN 2: COLOR DECONTAMINATION ---
// Color decontamination for edges
if (alpha > 0f && alpha < 1.0f) {
val grayIntensity = (fgR + fgG + fgB) / 3f
fgR = (fgR * alpha + grayIntensity * (1f - alpha)).toInt()
@@ -2066,27 +1871,30 @@ class MainActivity : AppCompatActivity() {
fgB = (fgB * alpha + grayIntensity * (1f - alpha)).toInt()
}
// --- THUẬT TOÁN 3: ALPHA BLENDING ---
val outR = (fgR * alpha + bgR * (1f - alpha)).toInt().coerceIn(0, 255)
val outG = (fgG * alpha + bgG * (1f - alpha)).toInt().coerceIn(0, 255)
val outB = (fgB * alpha + bgB * (1f - alpha)).toInt().coerceIn(0, 255)
outputPixels[idx] = android.graphics.Color.rgb(outR, outG, outB)
}
}
blurStage1?.recycle()
blurStage2?.recycle()
blurStage3?.recycle()
bgBitmap.recycle()
blurredBg?.recycle()
val outputBitmap = android.graphics.Bitmap.createBitmap(w, h, android.graphics.Bitmap.Config.ARGB_8888)
outputBitmap.setPixels(outputPixels, 0, w, 0, 0, w, h)
return outputBitmap
} catch (e: Exception) {
return capturedImage
}
}
} catch (e: Exception) {
// Ignore and fallback
}
if (blurIntensity > 0f) {
val radius = (25 * blurIntensity).toInt().coerceIn(1, 25)
return blurBitmap(capturedImage, radius)
}
return capturedImage
}
private fun blurAlphaMaskBox(arr: FloatArray, w: Int, h: Int, radius: Int) {
if (radius <= 0) return
@@ -2217,7 +2025,6 @@ class MainActivity : AppCompatActivity() {
cameraExecutor.shutdown()
try {
imageSegmenter?.close()
tfliteInterpreter?.close()
} catch (e: Exception) {
// Ignore
}
assets/models/midas.tflite
BIN
View File
Binary file not shown.