Skip to content

Commit

Permalink
Add English and French dialog to the sample application
Browse files Browse the repository at this point in the history
  • Loading branch information
Joss STUART committed Oct 28, 2020
1 parent c32a8fb commit a84e1e3
Show file tree
Hide file tree
Showing 10 changed files with 454 additions and 53 deletions.
2 changes: 1 addition & 1 deletion FaceMaskDetection/src/main/java/com/softbankrobotics/facemaskdetection/FaceMaskDetection.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class FaceMaskDetection(private val detector: FaceMaskDetector, private val came

private val detectionScope = CoroutineScope(
ThreadPoolExecutor(1, 1, 30, TimeUnit.SECONDS,
ArrayBlockingQueue(1), ThreadPoolExecutor.DiscardOldestPolicy()).asCoroutineDispatcher())
ArrayBlockingQueue(1), ThreadPoolExecutor.DiscardOldestPolicy()).asCoroutineDispatcher())

sealed class Message {
class FaceMaskDetect(
Expand Down
14 changes: 11 additions & 3 deletions app/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ android {
buildToolsVersion "29.0.3"

defaultConfig {
applicationId "com.softbankrobotics.peppermaskdetection"
applicationId "com.softbankrobotics.dx.peppermaskdetection"
minSdkVersion 23
targetSdkVersion 29
versionCode 6
versionName "1.3.2"
versionCode 9
versionName "2.3"

testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
}
Expand Down Expand Up @@ -61,4 +61,12 @@ dependencies {
// OpenCV
implementation project(":OpenCV")
implementation project(":FaceMaskDetection")

implementation 'com.aldebaran:qisdk-conversationalcontent-greetings:0.19.1-experimental-05'
implementation 'com.aldebaran:qisdk-conversationalcontent-robotabilities:0.19.1-experimental-05'
implementation 'com.aldebaran:qisdk-conversationalcontent-volumecontrol:0.19.1-experimental-05'
implementation 'com.aldebaran:qisdk-conversationalcontent-farewell:0.19.1-experimental-05'
implementation 'com.aldebaran:qisdk-conversationalcontent-datetime:0.19.1-experimental-05'
implementation 'com.aldebaran:qisdk-conversationalcontent-askrobotname:0.19.1-experimental-05'
implementation 'com.aldebaran:qisdk-conversationalcontent-conversationbasics:0.19.1-experimental-05'
}
231 changes: 207 additions & 24 deletions app/src/main/java/com/softbankrobotics/peppermaskdetection/MainActivity.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,37 @@ import com.aldebaran.qi.Future
import com.aldebaran.qi.sdk.QiContext
import com.aldebaran.qi.sdk.QiSDK
import com.aldebaran.qi.sdk.RobotLifecycleCallbacks
import com.aldebaran.qi.sdk.`object`.conversation.*
import com.aldebaran.qi.sdk.builder.QiChatbotBuilder
import com.aldebaran.qi.sdk.builder.TopicBuilder
import com.aldebaran.qi.sdk.conversationalcontentlibrary.askrobotname.AskRobotNameConversationalContent
import com.aldebaran.qi.sdk.conversationalcontentlibrary.base.AbstractConversationalContent
import com.aldebaran.qi.sdk.conversationalcontentlibrary.base.ConversationalContentChatBuilder
import com.aldebaran.qi.sdk.conversationalcontentlibrary.datetime.DateTimeConversationalContent
import com.aldebaran.qi.sdk.conversationalcontentlibrary.farewell.FarewellConversationalContent
import com.aldebaran.qi.sdk.conversationalcontentlibrary.greetings.GreetingsConversationalContent
import com.aldebaran.qi.sdk.conversationalcontentlibrary.robotabilities.RobotAbilitiesConversationalContent
import com.aldebaran.qi.sdk.conversationalcontentlibrary.volumecontrol.VolumeControlConversationalContent
import com.aldebaran.qi.sdk.design.activity.RobotActivity
import com.aldebaran.qi.sdk.design.activity.conversationstatus.SpeechBarDisplayStrategy
import com.softbankrobotics.facemaskdetection.FaceMaskDetection
import com.softbankrobotics.facemaskdetection.capturer.BottomCameraCapturer
import com.softbankrobotics.facemaskdetection.detector.AizooFaceMaskDetector
import com.softbankrobotics.facemaskdetection.capturer.TopCameraCapturer
import com.softbankrobotics.facemaskdetection.detector.AizooFaceMaskDetector
import com.softbankrobotics.facemaskdetection.detector.FaceMaskDetector
import com.softbankrobotics.facemaskdetection.utils.OpenCVUtils
import com.softbankrobotics.facemaskdetection.utils.TAG
import kotlinx.android.synthetic.main.activity_main.*

class MainActivity : RobotActivity(), RobotLifecycleCallbacks {

private val useTopCamera = false
private var qiChatbot: QiChatbot? = null
private var chatFuture: Future<Void>? = null
private val useTopCamera = true
private var shouldBeRecognizing = false

override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setSpeechBarDisplayStrategy(SpeechBarDisplayStrategy.IMMERSIVE)
//setSpeechBarDisplayStrategy(SpeechBarDisplayStrategy.IMMERSIVE)
setContentView(R.layout.activity_main)
clearFaces()
if (useTopCamera || cameraPermissionAlreadyGranted()) {
Expand All @@ -45,6 +58,8 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {

public override fun onPause() {
super.onPause()
Log.i(TAG, "onPause")
shouldBeRecognizing = false
detectionFuture?.requestCancellation()
detectionFuture = null
}
Expand All @@ -53,6 +68,7 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
super.onResume()
OpenCVUtils.loadOpenCV(this)
clearFaces()
shouldBeRecognizing = true
startDetecting()
}

Expand All @@ -73,14 +89,17 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {

private fun requestPermissionForCamera() {
if (ActivityCompat.shouldShowRequestPermissionRationale(this, Manifest.permission.CAMERA)) {
Toast.makeText(this,
Toast.makeText(
this,
R.string.permissions_needed,
Toast.LENGTH_LONG).show()
Toast.LENGTH_LONG
).show()
} else {
ActivityCompat.requestPermissions(
this,
arrayOf(Manifest.permission.CAMERA),
CAMERA_PERMISSION_REQUEST_CODE)
CAMERA_PERMISSION_REQUEST_CODE
)
}
}

Expand All @@ -89,9 +108,11 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
return resultCamera == PackageManager.PERMISSION_GRANTED
}

override fun onRequestPermissionsResult(requestCode: Int,
permissions: Array<String>,
grantResults: IntArray) {
override fun onRequestPermissionsResult(
requestCode: Int,
permissions: Array<String>,
grantResults: IntArray
) {
if (requestCode == CAMERA_PERMISSION_REQUEST_CODE) {
var cameraPermissionGranted = true

Expand All @@ -102,9 +123,11 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
if (cameraPermissionGranted) {
QiSDK.register(this, this)
} else {
Toast.makeText(this,
Toast.makeText(
this,
R.string.permissions_needed,
Toast.LENGTH_LONG).show()
Toast.LENGTH_LONG
).show()
}
}
}
Expand All @@ -116,17 +139,18 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
inner class FacesForDisplay(rawFaces: List<FaceMaskDetector.DetectedFace>) {
// Choose the "main" focused faced, which is either the biggest or, when there are a lot of
// people, the one in the middle.
val mainFace : FaceMaskDetector.DetectedFace? = when {
val mainFace: FaceMaskDetector.DetectedFace? = when {
rawFaces.size >= 5 -> rawFaces[2]
rawFaces.size == 4 -> rawFaces.subList(1, 3).maxBy { it.size() }
else -> rawFaces.maxBy { it.size() }
}
private val mainFaceIndex = rawFaces.indexOf(mainFace)

// Set the other faces relatively
val leftFarFace : FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex - 2)
val leftNearFace : FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex - 1)
val rightNearFace : FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex + 1)
val rightFarFace : FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex + 2)
val leftFarFace: FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex - 2)
val leftNearFace: FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex - 1)
val rightNearFace: FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex + 1)
val rightFarFace: FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex + 2)
}

private fun setFaces(faces: List<FaceMaskDetector.DetectedFace>) {
Expand All @@ -150,7 +174,11 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
}
}

private fun setFace(card: View, face: FaceMaskDetector.DetectedFace?, hideIfEmpty : Boolean = true) {
private fun setFace(
card: View,
face: FaceMaskDetector.DetectedFace?,
hideIfEmpty: Boolean = true
) {
if (hideIfEmpty && face == null) {
card.visibility = View.INVISIBLE
} else {
Expand All @@ -167,7 +195,8 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
resources.getColor(R.color.colorNoMaskDetected, null)
}
circle.setColorFilter(color)
label.text = resources.getString(if(face.hasMask) R.string.mask else R.string.no_mask)
label.text =
resources.getString(if (face.hasMask) R.string.mask else R.string.no_mask)
} else {
photo.visibility = View.INVISIBLE
circle.setColorFilter(resources.getColor(R.color.colorNobody, null))
Expand All @@ -176,35 +205,189 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
}
}


/**********************
* Greetings
**********************/

var engaged = false
var sawNobodyCount = 0

private fun jumpToBookmark(bookmark: Bookmark?) {
bookmark?.let {
qiChatbot?.async()?.goToBookmark(
it,
AutonomousReactionImportance.HIGH,
AutonomousReactionValidity.DELAYABLE
)
}
}

var lastSawWithoutMask = false
var annoyance = 0

var lastMentionedPeopleNum = 0
var worthMentioningPeopleCounter = 0

private fun updateState(seesWithMask: Boolean, seesWithoutMask: Boolean, numPeople: Int) {
val seesSomebody = seesWithMask || seesWithoutMask
Log.w(
TAG,
"DBG updating state wMask=${seesWithMask} noMask=${seesWithoutMask} engaged=${engaged}"
)

if (engaged) {
// Update status when already engaged
if (seesSomebody) {
sawNobodyCount = 0 // Stay engaged
// See if they put on a mask
var saidSomething = false

// See if it's worth mentioning people putting on masks or taking them off
if (seesWithoutMask == lastSawWithoutMask) {
annoyance = 0
} else {
// Something changed !
annoyance += 1
if (annoyance >= 2) {
annoyance = 0
lastSawWithoutMask = seesWithoutMask
saidSomething = true
if (seesWithoutMask) {
if (seesWithMask) {
jumpToBookmark(newWithoutMaskBookmark)
} else {
jumpToBookmark(tookOffMaskBookmark)
}
} else {
jumpToBookmark(putOnMaskBookmark)
}
lastMentionedPeopleNum = numPeople
worthMentioningPeopleCounter = 0
}
}

// See if it's worth mentioning a lot of people
if (numPeople > lastMentionedPeopleNum) {
if (worthMentioningPeopleCounter > 2) {
lastMentionedPeopleNum = numPeople
worthMentioningPeopleCounter = 0
jumpToBookmark(manyPeopleBookmark)
} else {
worthMentioningPeopleCounter += 1
}
} else {
lastMentionedPeopleNum = numPeople
worthMentioningPeopleCounter = 0
}

} else {
sawNobodyCount += 1
if (sawNobodyCount > 2) {
engaged = false
//chat?.removeAllOnStartedListeners()
//chatFuture?.cancel(false)
chatFuture = null
lastSawWithoutMask = false
annoyance = 0
}
}
} else if (seesSomebody) {
engaged = true
chat?.let { chat ->
if (seesWithoutMask) {
jumpToBookmark(noMaskBookmark)
} else {
jumpToBookmark(maskBookmark)
}
lastSawWithoutMask = seesWithoutMask
annoyance = 0
}
}
}


/**********************
* Robot Lifecycle
**********************/


private fun startDetecting() {
detectionFuture = detection?.start { faces ->
// Filter and sort the faces so that they're left to right, with no uncertain or
// non-unique results
// Filter and sort the faces so that they're left to right and certain enough
val sortedFaces = faces
.filter { (it.confidence > 0.5)}
.filter { (it.confidence > 0.5) }
.sortedBy { -it.bb.left }
Log.v(TAG, "Filtered faces ${faces.size}, -> ${sortedFaces.size}")
Log.i(TAG, "Filtered faces ${faces.size}, -> ${sortedFaces.size}")
setFaces(sortedFaces)
// Now update the logic
val seesWithMask = sortedFaces.any { it.hasMask }
val seesWithoutMask = sortedFaces.any { !it.hasMask }
val numPeople = sortedFaces.size
updateState(seesWithMask, seesWithoutMask, numPeople)
}
detectionFuture?.thenConsume {
Log.i(TAG, "Detection future has finished: success=${it.isSuccess}, cancelled=${it.isCancelled}")
Log.i(
TAG,
"Detection future has finished: success=${it.isSuccess}, cancelled=${it.isCancelled}"
)
if (shouldBeRecognizing) {
Log.w(TAG, "Stopped, but it shouldn't have - starting it again")
startDetecting()
}
}
}

var chat: Chat? = null
var maskBookmark: Bookmark? = null
var noMaskBookmark: Bookmark? = null
var tookOffMaskBookmark: Bookmark? = null
var putOnMaskBookmark: Bookmark? = null
var newWithoutMaskBookmark: Bookmark? = null
var manyPeopleBookmark: Bookmark? = null

override fun onRobotFocusGained(qiContext: QiContext) {
Log.i(TAG, "onRobotFocusGained")
if (chat == null) {
val topic = TopicBuilder.with(qiContext).withResource(R.raw.chat).build()
val qiChatbot = QiChatbotBuilder.with(qiContext).withTopic(topic).build()
this.qiChatbot = qiChatbot
maskBookmark = topic.bookmarks["GREETING_MASK"]
noMaskBookmark = topic.bookmarks["GREETING_NO_MASK"]
tookOffMaskBookmark = topic.bookmarks["TOOK_OFF_MASK"]
putOnMaskBookmark = topic.bookmarks["PUT_ON_MASK"]
newWithoutMaskBookmark = topic.bookmarks["NEW_WITHOUT_MASK"]
manyPeopleBookmark = topic.bookmarks["MANY_PEOPLE"]

val conversationalContents: List<AbstractConversationalContent> = listOf(
GreetingsConversationalContent(),
FarewellConversationalContent(),
AskRobotNameConversationalContent(),
DateTimeConversationalContent(),
RobotAbilitiesConversationalContent(),
VolumeControlConversationalContent()
)

chat = ConversationalContentChatBuilder.with(qiContext)
.withChatbot(qiChatbot)
.withConversationalContents(conversationalContents)
.build()
chat?.listeningBodyLanguage = BodyLanguageOption.DISABLED
}

Log.i(TAG, "Initialised chat")

val capturer = if (useTopCamera) {
TopCameraCapturer(qiContext)
} else {
BottomCameraCapturer(this, this)
}
val detector = AizooFaceMaskDetector(this)
detection = FaceMaskDetection(detector, capturer)
shouldBeRecognizing = true
startDetecting()
Log.i(TAG, "Starting chat")
chatFuture = chat?.async()?.run()
}

override fun onRobotFocusLost() {
Expand Down
Binary file added app/src/main/res/drawable/sbrlogo.jpg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading

0 comments on commit a84e1e3

Please sign in to comment.