diff --git a/FaceMaskDetection/src/main/java/com/softbankrobotics/facemaskdetection/FaceMaskDetection.kt b/FaceMaskDetection/src/main/java/com/softbankrobotics/facemaskdetection/FaceMaskDetection.kt index 290de2f..a7e417e 100644 --- a/FaceMaskDetection/src/main/java/com/softbankrobotics/facemaskdetection/FaceMaskDetection.kt +++ b/FaceMaskDetection/src/main/java/com/softbankrobotics/facemaskdetection/FaceMaskDetection.kt @@ -28,7 +28,7 @@ class FaceMaskDetection(private val detector: FaceMaskDetector, private val came private val detectionScope = CoroutineScope( ThreadPoolExecutor(1, 1, 30, TimeUnit.SECONDS, - ArrayBlockingQueue(1), ThreadPoolExecutor.DiscardOldestPolicy()).asCoroutineDispatcher()) + ArrayBlockingQueue(1), ThreadPoolExecutor.DiscardOldestPolicy()).asCoroutineDispatcher()) sealed class Message { class FaceMaskDetect( diff --git a/app/build.gradle b/app/build.gradle index db20e18..ddac0c5 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -7,11 +7,11 @@ android { buildToolsVersion "29.0.3" defaultConfig { - applicationId "com.softbankrobotics.peppermaskdetection" + applicationId "com.softbankrobotics.dx.peppermaskdetection" minSdkVersion 23 targetSdkVersion 29 - versionCode 6 - versionName "1.3.2" + versionCode 9 + versionName "2.3" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" } @@ -61,4 +61,12 @@ dependencies { // OpenCV implementation project(":OpenCV") implementation project(":FaceMaskDetection") + + implementation 'com.aldebaran:qisdk-conversationalcontent-greetings:0.19.1-experimental-05' + implementation 'com.aldebaran:qisdk-conversationalcontent-robotabilities:0.19.1-experimental-05' + implementation 'com.aldebaran:qisdk-conversationalcontent-volumecontrol:0.19.1-experimental-05' + implementation 'com.aldebaran:qisdk-conversationalcontent-farewell:0.19.1-experimental-05' + implementation 'com.aldebaran:qisdk-conversationalcontent-datetime:0.19.1-experimental-05' + implementation 'com.aldebaran:qisdk-conversationalcontent-askrobotname:0.19.1-experimental-05' + implementation 'com.aldebaran:qisdk-conversationalcontent-conversationbasics:0.19.1-experimental-05' } diff --git a/app/src/main/java/com/softbankrobotics/peppermaskdetection/MainActivity.kt b/app/src/main/java/com/softbankrobotics/peppermaskdetection/MainActivity.kt index 54ab031..82ef9f2 100644 --- a/app/src/main/java/com/softbankrobotics/peppermaskdetection/MainActivity.kt +++ b/app/src/main/java/com/softbankrobotics/peppermaskdetection/MainActivity.kt @@ -14,12 +14,22 @@ import com.aldebaran.qi.Future import com.aldebaran.qi.sdk.QiContext import com.aldebaran.qi.sdk.QiSDK import com.aldebaran.qi.sdk.RobotLifecycleCallbacks +import com.aldebaran.qi.sdk.`object`.conversation.* +import com.aldebaran.qi.sdk.builder.QiChatbotBuilder +import com.aldebaran.qi.sdk.builder.TopicBuilder +import com.aldebaran.qi.sdk.conversationalcontentlibrary.askrobotname.AskRobotNameConversationalContent +import com.aldebaran.qi.sdk.conversationalcontentlibrary.base.AbstractConversationalContent +import com.aldebaran.qi.sdk.conversationalcontentlibrary.base.ConversationalContentChatBuilder +import com.aldebaran.qi.sdk.conversationalcontentlibrary.datetime.DateTimeConversationalContent +import com.aldebaran.qi.sdk.conversationalcontentlibrary.farewell.FarewellConversationalContent +import com.aldebaran.qi.sdk.conversationalcontentlibrary.greetings.GreetingsConversationalContent +import com.aldebaran.qi.sdk.conversationalcontentlibrary.robotabilities.RobotAbilitiesConversationalContent +import com.aldebaran.qi.sdk.conversationalcontentlibrary.volumecontrol.VolumeControlConversationalContent import com.aldebaran.qi.sdk.design.activity.RobotActivity -import com.aldebaran.qi.sdk.design.activity.conversationstatus.SpeechBarDisplayStrategy import com.softbankrobotics.facemaskdetection.FaceMaskDetection import com.softbankrobotics.facemaskdetection.capturer.BottomCameraCapturer -import com.softbankrobotics.facemaskdetection.detector.AizooFaceMaskDetector import com.softbankrobotics.facemaskdetection.capturer.TopCameraCapturer +import com.softbankrobotics.facemaskdetection.detector.AizooFaceMaskDetector import com.softbankrobotics.facemaskdetection.detector.FaceMaskDetector import com.softbankrobotics.facemaskdetection.utils.OpenCVUtils import com.softbankrobotics.facemaskdetection.utils.TAG @@ -27,11 +37,14 @@ import kotlinx.android.synthetic.main.activity_main.* class MainActivity : RobotActivity(), RobotLifecycleCallbacks { - private val useTopCamera = false + private var qiChatbot: QiChatbot? = null + private var chatFuture: Future? = null + private val useTopCamera = true + private var shouldBeRecognizing = false override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - setSpeechBarDisplayStrategy(SpeechBarDisplayStrategy.IMMERSIVE) + //setSpeechBarDisplayStrategy(SpeechBarDisplayStrategy.IMMERSIVE) setContentView(R.layout.activity_main) clearFaces() if (useTopCamera || cameraPermissionAlreadyGranted()) { @@ -45,6 +58,8 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks { public override fun onPause() { super.onPause() + Log.i(TAG, "onPause") + shouldBeRecognizing = false detectionFuture?.requestCancellation() detectionFuture = null } @@ -53,6 +68,7 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks { super.onResume() OpenCVUtils.loadOpenCV(this) clearFaces() + shouldBeRecognizing = true startDetecting() } @@ -73,14 +89,17 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks { private fun requestPermissionForCamera() { if (ActivityCompat.shouldShowRequestPermissionRationale(this, Manifest.permission.CAMERA)) { - Toast.makeText(this, + Toast.makeText( + this, R.string.permissions_needed, - Toast.LENGTH_LONG).show() + Toast.LENGTH_LONG + ).show() } else { ActivityCompat.requestPermissions( this, arrayOf(Manifest.permission.CAMERA), - CAMERA_PERMISSION_REQUEST_CODE) + CAMERA_PERMISSION_REQUEST_CODE + ) } } @@ -89,9 +108,11 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks { return resultCamera == PackageManager.PERMISSION_GRANTED } - override fun onRequestPermissionsResult(requestCode: Int, - permissions: Array, - grantResults: IntArray) { + override fun onRequestPermissionsResult( + requestCode: Int, + permissions: Array, + grantResults: IntArray + ) { if (requestCode == CAMERA_PERMISSION_REQUEST_CODE) { var cameraPermissionGranted = true @@ -102,9 +123,11 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks { if (cameraPermissionGranted) { QiSDK.register(this, this) } else { - Toast.makeText(this, + Toast.makeText( + this, R.string.permissions_needed, - Toast.LENGTH_LONG).show() + Toast.LENGTH_LONG + ).show() } } } @@ -116,17 +139,18 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks { inner class FacesForDisplay(rawFaces: List) { // Choose the "main" focused faced, which is either the biggest or, when there are a lot of // people, the one in the middle. - val mainFace : FaceMaskDetector.DetectedFace? = when { + val mainFace: FaceMaskDetector.DetectedFace? = when { rawFaces.size >= 5 -> rawFaces[2] rawFaces.size == 4 -> rawFaces.subList(1, 3).maxBy { it.size() } else -> rawFaces.maxBy { it.size() } } private val mainFaceIndex = rawFaces.indexOf(mainFace) + // Set the other faces relatively - val leftFarFace : FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex - 2) - val leftNearFace : FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex - 1) - val rightNearFace : FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex + 1) - val rightFarFace : FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex + 2) + val leftFarFace: FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex - 2) + val leftNearFace: FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex - 1) + val rightNearFace: FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex + 1) + val rightFarFace: FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex + 2) } private fun setFaces(faces: List) { @@ -150,7 +174,11 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks { } } - private fun setFace(card: View, face: FaceMaskDetector.DetectedFace?, hideIfEmpty : Boolean = true) { + private fun setFace( + card: View, + face: FaceMaskDetector.DetectedFace?, + hideIfEmpty: Boolean = true + ) { if (hideIfEmpty && face == null) { card.visibility = View.INVISIBLE } else { @@ -167,7 +195,8 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks { resources.getColor(R.color.colorNoMaskDetected, null) } circle.setColorFilter(color) - label.text = resources.getString(if(face.hasMask) R.string.mask else R.string.no_mask) + label.text = + resources.getString(if (face.hasMask) R.string.mask else R.string.no_mask) } else { photo.visibility = View.INVISIBLE circle.setColorFilter(resources.getColor(R.color.colorNobody, null)) @@ -176,27 +205,178 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks { } } + + /********************** + * Greetings + **********************/ + + var engaged = false + var sawNobodyCount = 0 + + private fun jumpToBookmark(bookmark: Bookmark?) { + bookmark?.let { + qiChatbot?.async()?.goToBookmark( + it, + AutonomousReactionImportance.HIGH, + AutonomousReactionValidity.DELAYABLE + ) + } + } + + var lastSawWithoutMask = false + var annoyance = 0 + + var lastMentionedPeopleNum = 0 + var worthMentioningPeopleCounter = 0 + + private fun updateState(seesWithMask: Boolean, seesWithoutMask: Boolean, numPeople: Int) { + val seesSomebody = seesWithMask || seesWithoutMask + Log.w( + TAG, + "DBG updating state wMask=${seesWithMask} noMask=${seesWithoutMask} engaged=${engaged}" + ) + + if (engaged) { + // Update status when already engaged + if (seesSomebody) { + sawNobodyCount = 0 // Stay engaged + // See if they put on a mask + var saidSomething = false + + // See if it's worth mentioning people putting on masks or taking them off + if (seesWithoutMask == lastSawWithoutMask) { + annoyance = 0 + } else { + // Something changed ! + annoyance += 1 + if (annoyance >= 2) { + annoyance = 0 + lastSawWithoutMask = seesWithoutMask + saidSomething = true + if (seesWithoutMask) { + if (seesWithMask) { + jumpToBookmark(newWithoutMaskBookmark) + } else { + jumpToBookmark(tookOffMaskBookmark) + } + } else { + jumpToBookmark(putOnMaskBookmark) + } + lastMentionedPeopleNum = numPeople + worthMentioningPeopleCounter = 0 + } + } + + // See if it's worth mentioning a lot of people + if (numPeople > lastMentionedPeopleNum) { + if (worthMentioningPeopleCounter > 2) { + lastMentionedPeopleNum = numPeople + worthMentioningPeopleCounter = 0 + jumpToBookmark(manyPeopleBookmark) + } else { + worthMentioningPeopleCounter += 1 + } + } else { + lastMentionedPeopleNum = numPeople + worthMentioningPeopleCounter = 0 + } + + } else { + sawNobodyCount += 1 + if (sawNobodyCount > 2) { + engaged = false + //chat?.removeAllOnStartedListeners() + //chatFuture?.cancel(false) + chatFuture = null + lastSawWithoutMask = false + annoyance = 0 + } + } + } else if (seesSomebody) { + engaged = true + chat?.let { chat -> + if (seesWithoutMask) { + jumpToBookmark(noMaskBookmark) + } else { + jumpToBookmark(maskBookmark) + } + lastSawWithoutMask = seesWithoutMask + annoyance = 0 + } + } + } + + /********************** * Robot Lifecycle **********************/ + private fun startDetecting() { detectionFuture = detection?.start { faces -> - // Filter and sort the faces so that they're left to right, with no uncertain or - // non-unique results + // Filter and sort the faces so that they're left to right and certain enough val sortedFaces = faces - .filter { (it.confidence > 0.5)} + .filter { (it.confidence > 0.5) } .sortedBy { -it.bb.left } - Log.v(TAG, "Filtered faces ${faces.size}, -> ${sortedFaces.size}") + Log.i(TAG, "Filtered faces ${faces.size}, -> ${sortedFaces.size}") setFaces(sortedFaces) + // Now update the logic + val seesWithMask = sortedFaces.any { it.hasMask } + val seesWithoutMask = sortedFaces.any { !it.hasMask } + val numPeople = sortedFaces.size + updateState(seesWithMask, seesWithoutMask, numPeople) } detectionFuture?.thenConsume { - Log.i(TAG, "Detection future has finished: success=${it.isSuccess}, cancelled=${it.isCancelled}") + Log.i( + TAG, + "Detection future has finished: success=${it.isSuccess}, cancelled=${it.isCancelled}" + ) + if (shouldBeRecognizing) { + Log.w(TAG, "Stopped, but it shouldn't have - starting it again") + startDetecting() + } } } + var chat: Chat? = null + var maskBookmark: Bookmark? = null + var noMaskBookmark: Bookmark? = null + var tookOffMaskBookmark: Bookmark? = null + var putOnMaskBookmark: Bookmark? = null + var newWithoutMaskBookmark: Bookmark? = null + var manyPeopleBookmark: Bookmark? = null + override fun onRobotFocusGained(qiContext: QiContext) { Log.i(TAG, "onRobotFocusGained") + if (chat == null) { + val topic = TopicBuilder.with(qiContext).withResource(R.raw.chat).build() + val qiChatbot = QiChatbotBuilder.with(qiContext).withTopic(topic).build() + this.qiChatbot = qiChatbot + maskBookmark = topic.bookmarks["GREETING_MASK"] + noMaskBookmark = topic.bookmarks["GREETING_NO_MASK"] + tookOffMaskBookmark = topic.bookmarks["TOOK_OFF_MASK"] + putOnMaskBookmark = topic.bookmarks["PUT_ON_MASK"] + newWithoutMaskBookmark = topic.bookmarks["NEW_WITHOUT_MASK"] + manyPeopleBookmark = topic.bookmarks["MANY_PEOPLE"] + + val conversationalContents: List = listOf( + GreetingsConversationalContent(), + FarewellConversationalContent(), + AskRobotNameConversationalContent(), + DateTimeConversationalContent(), + RobotAbilitiesConversationalContent(), + VolumeControlConversationalContent() + ) + + chat = ConversationalContentChatBuilder.with(qiContext) + .withChatbot(qiChatbot) + .withConversationalContents(conversationalContents) + .build() + chat?.listeningBodyLanguage = BodyLanguageOption.DISABLED + } + + Log.i(TAG, "Initialised chat") + val capturer = if (useTopCamera) { TopCameraCapturer(qiContext) } else { @@ -204,7 +384,10 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks { } val detector = AizooFaceMaskDetector(this) detection = FaceMaskDetection(detector, capturer) + shouldBeRecognizing = true startDetecting() + Log.i(TAG, "Starting chat") + chatFuture = chat?.async()?.run() } override fun onRobotFocusLost() { diff --git a/app/src/main/res/drawable/sbrlogo.jpg b/app/src/main/res/drawable/sbrlogo.jpg new file mode 100644 index 0000000..af4dbe4 Binary files /dev/null and b/app/src/main/res/drawable/sbrlogo.jpg differ diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index 2cd45bd..80e953e 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -4,32 +4,55 @@ xmlns:tools="http://schemas.android.com/tools" android:layout_width="match_parent" android:layout_height="match_parent" - android:layout_gravity="center" + android:layout_gravity="bottom" android:background="#F0F0F0" - android:foregroundGravity="center" + android:foregroundGravity="bottom" tools:context=".MainActivity"> - - - - + + + + + + + + + +