hlrc / server / src / Arbiter.cpp @ ebcb799f
History | View | Annotate | Download (11.429 KB)
1 | 0c286af0 | Simon Schulz | /*
|
---|---|---|---|
2 | f150aab5 | Robert Haschke | * This file is part of hlrc_server
|
3 | *
|
||
4 | * Copyright(c) sschulz <AT> techfak.uni-bielefeld.de
|
||
5 | * http://opensource.cit-ec.de/projects/hlrc_server
|
||
6 | *
|
||
7 | * This file may be licensed under the terms of the
|
||
8 | * GNU General Public License Version 3 (the ``GPL''),
|
||
9 | * or (at your option) any later version.
|
||
10 | *
|
||
11 | * Software distributed under the License is distributed
|
||
12 | * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
|
||
13 | * express or implied. See the GPL for the specific language
|
||
14 | * governing rights and limitations.
|
||
15 | *
|
||
16 | * You should have received a copy of the GPL along with this
|
||
17 | * program. If not, go to http://www.gnu.org/licenses/gpl.html
|
||
18 | * or write to the Free Software Foundation, Inc.,
|
||
19 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||
20 | *
|
||
21 | * The development of this software was supported by the
|
||
22 | * Excellence Cluster EXC 277 Cognitive Interaction Technology.
|
||
23 | * The Excellence Cluster EXC 277 is a grant of the Deutsche
|
||
24 | * Forschungsgemeinschaft (DFG) in the context of the German
|
||
25 | * Excellence Initiative.
|
||
26 | *
|
||
27 | */
|
||
28 | 0c286af0 | Simon Schulz | |
29 | #include "Arbiter.h" |
||
30 | #include "AudioPlayerLibAO.h" |
||
31 | #include "AudioPlayerRSB.h" |
||
32 | #include <boost/algorithm/string/predicate.hpp> |
||
33 | 42084118 | Robert Haschke | #include <mutex> |
34 | #include <thread> |
||
35 | 90fa4b4e | Robert Haschke | #include <iostream> |
36 | 0c286af0 | Simon Schulz | |
37 | using namespace boost; |
||
38 | using namespace std; |
||
39 | |||
40 | f150aab5 | Robert Haschke | Arbiter::Arbiter(std::string audio_output) {
|
41 | // initialize audio player:
|
||
42 | if (iequals(audio_output, "none")) { |
||
43 | // allow none for no sound output
|
||
44 | audio_player = NULL;
|
||
45 | } |
||
46 | else if (iequals(audio_output.substr(0, 3), "rsb")) { |
||
47 | #ifdef RSB_SUPPORT
|
||
48 | audio_player = new AudioPlayerRSB(audio_output);
|
||
49 | #else
|
||
50 | printf("> ERROR: hlc is compiled without RSB support, RSB audio transport not available, defaulting to libao (default "
|
||
51 | "output!)\n");
|
||
52 | audio_player = new AudioPlayerLibAO(""); |
||
53 | #endif
|
||
54 | } |
||
55 | else {
|
||
56 | audio_player = new AudioPlayerLibAO(audio_output);
|
||
57 | } |
||
58 | |||
59 | // set default emotion:
|
||
60 | emotion_config_current.init_sad(); |
||
61 | emotion_config_current.set_duration(2000);
|
||
62 | emotion_config_default.init_neutral(); |
||
63 | |||
64 | gaze_state_animation_restart = true;
|
||
65 | emotion_target = &emotion_config_default; |
||
66 | |||
67 | 482adb6d | Robert Haschke | utterance = std::shared_ptr<Utterance>(new Utterance());
|
68 | 0c286af0 | Simon Schulz | } |
69 | |||
70 | f150aab5 | Robert Haschke | Arbiter::~Arbiter() { |
71 | 0c286af0 | Simon Schulz | } |
72 | |||
73 | f150aab5 | Robert Haschke | void Arbiter::set_default_emotion(EmotionState e) {
|
74 | // lock access
|
||
75 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(emotion_config_default_mutex);
|
76 | 0c286af0 | Simon Schulz | |
77 | f150aab5 | Robert Haschke | // update our config, ignore duration as this is the default emotion
|
78 | emotion_config_default.select_config(e.value); |
||
79 | emotion_target = &emotion_config_default; |
||
80 | efbcb710 | sschulz | |
81 | f150aab5 | Robert Haschke | gaze_state_animation_restart = true;
|
82 | 0c286af0 | Simon Schulz | |
83 | f150aab5 | Robert Haschke | printf("> stored default emotion\n");
|
84 | 0c286af0 | Simon Schulz | } |
85 | |||
86 | f150aab5 | Robert Haschke | void Arbiter::set_current_emotion(EmotionState e) {
|
87 | // lock access
|
||
88 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(emotion_config_current_mutex);
|
89 | 0c286af0 | Simon Schulz | |
90 | f150aab5 | Robert Haschke | // this will set a emotion override for a given time, after the timeout
|
91 | // we will return to the default emotion state:
|
||
92 | emotion_config_current.select_config(e.value); |
||
93 | emotion_config_current.set_duration(e.duration); |
||
94 | 0c286af0 | Simon Schulz | |
95 | f150aab5 | Robert Haschke | emotion_target = &emotion_config_current; |
96 | gaze_state_animation_restart = true;
|
||
97 | efbcb710 | sschulz | |
98 | f150aab5 | Robert Haschke | printf("> stored current emotion (val=%d, duration = %dms)\n", e.value, e.duration);
|
99 | 0c286af0 | Simon Schulz | } |
100 | |||
101 | f150aab5 | Robert Haschke | void Arbiter::set_mouth_config(MouthConfig m) {
|
102 | // lock access
|
||
103 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(mouth_config_override_mutex);
|
104 | 0c286af0 | Simon Schulz | |
105 | f150aab5 | Robert Haschke | mouth_config = m; |
106 | 0c286af0 | Simon Schulz | } |
107 | |||
108 | f150aab5 | Robert Haschke | void Arbiter::set_gaze_target(humotion::GazeState target) {
|
109 | // lock access
|
||
110 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(requested_gaze_state_mutex);
|
111 | 0c286af0 | Simon Schulz | |
112 | f150aab5 | Robert Haschke | requested_gaze_state = target; |
113 | 0c286af0 | Simon Schulz | } |
114 | |||
115 | f150aab5 | Robert Haschke | void Arbiter::set_mouth_target(humotion::MouthState target) {
|
116 | // lock access
|
||
117 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(requested_mouth_state_mutex);
|
118 | 0c286af0 | Simon Schulz | |
119 | f150aab5 | Robert Haschke | requested_mouth_state = target; |
120 | 0c286af0 | Simon Schulz | } |
121 | |||
122 | //! note: this is blocking!
|
||
123 | 482adb6d | Robert Haschke | void Arbiter::play_animation(std::shared_ptr<Animation> incoming_animation) {
|
124 | f150aab5 | Robert Haschke | // incoming animation, check if this would conflict with any pending animations:
|
125 | // lock access & iterate over vector:
|
||
126 | 42084118 | Robert Haschke | std::unique_lock<std::mutex> lock_av(active_animation_vector_mutex); |
127 | f150aab5 | Robert Haschke | active_animation_vector_t::iterator it; |
128 | for (it = active_animation_vector.begin(); it < active_animation_vector.end(); it++) {
|
||
129 | 482adb6d | Robert Haschke | std::shared_ptr<Animation> current_ani = *it; |
130 | f150aab5 | Robert Haschke | // check if the running animation collides with the incoming animation:
|
131 | if (current_ani->collides_with(incoming_animation.get())) {
|
||
132 | // this would fail, we can not play this animation right now!
|
||
133 | throw runtime_error("incoming animation collides with unfinished active animation"); |
||
134 | return;
|
||
135 | } |
||
136 | } |
||
137 | |||
138 | // ok, fine. no other animations are active that would collide with the new request
|
||
139 | // thus it is safe to enqueue this item:
|
||
140 | active_animation_vector.push_back(incoming_animation); |
||
141 | |||
142 | // unlock mutex
|
||
143 | lock_av.unlock(); |
||
144 | |||
145 | // start playback!
|
||
146 | incoming_animation->start(); |
||
147 | |||
148 | // block until ani was played back
|
||
149 | while (incoming_animation->is_active()) {
|
||
150 | e8a50634 | Robert Haschke | std::this_thread::sleep_for(std::chrono::milliseconds(1)); // save cpu cycles |
151 | f150aab5 | Robert Haschke | } |
152 | |||
153 | // ok, it finished. we can safely remove it now:
|
||
154 | lock_av.lock(); |
||
155 | for (it = active_animation_vector.begin(); it < active_animation_vector.end();) {
|
||
156 | 482adb6d | Robert Haschke | std::shared_ptr<Animation> current_ani = *it; |
157 | f150aab5 | Robert Haschke | if (*it == incoming_animation) {
|
158 | // printf(">match -> remove incoming ani again\n");
|
||
159 | it = active_animation_vector.erase(it); |
||
160 | } |
||
161 | else {
|
||
162 | ++it; |
||
163 | } |
||
164 | } |
||
165 | 0c286af0 | Simon Schulz | } |
166 | |||
167 | f150aab5 | Robert Haschke | void Arbiter::speak(
|
168 | 482adb6d | Robert Haschke | std::shared_ptr<Utterance> u) { //, ao_sample_format audio_format, char *audio_data, unsigned int audio_len){
|
169 | f150aab5 | Robert Haschke | // lock audio playback as such:
|
170 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock_audio(audio_player_mutex);
|
171 | f150aab5 | Robert Haschke | |
172 | 42084118 | Robert Haschke | { |
173 | // lock utterance & store data:
|
||
174 | const std::lock_guard<std::mutex> lock(utterance_mutex);
|
||
175 | utterance = u; |
||
176 | } |
||
177 | f150aab5 | Robert Haschke | |
178 | // start audio playback, this function returns once we started playback
|
||
179 | if (audio_player == NULL) { |
||
180 | printf("> audio_player disabled, not speaking '%s'\n", u.get()->get_text().c_str());
|
||
181 | } |
||
182 | else {
|
||
183 | audio_player->play(utterance->get_audio_data()); |
||
184 | } |
||
185 | utterance->start_playback(); |
||
186 | |||
187 | // wait until audio playback was finished:
|
||
188 | if (audio_player != NULL) { |
||
189 | while (audio_player->is_playing()) {
|
||
190 | // save some cpu cycles:
|
||
191 | e8a50634 | Robert Haschke | std::this_thread::sleep_for(std::chrono::milliseconds(1)); // 1ms |
192 | f150aab5 | Robert Haschke | } |
193 | } |
||
194 | |||
195 | // in case the audio output fails, we end up here before the utterance is finished.
|
||
196 | // so check now if the utterance finished as well:
|
||
197 | while (utterance->is_playing()) {
|
||
198 | // save some cpu cycles:
|
||
199 | e8a50634 | Robert Haschke | std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
200 | f150aab5 | Robert Haschke | } |
201 | 0c286af0 | Simon Schulz | } |
202 | |||
203 | f150aab5 | Robert Haschke | bool Arbiter::speak_active() {
|
204 | if (audio_player == NULL) { |
||
205 | return false; |
||
206 | } |
||
207 | else {
|
||
208 | return audio_player->is_playing();
|
||
209 | } |
||
210 | 0c286af0 | Simon Schulz | } |
211 | |||
212 | f150aab5 | Robert Haschke | void Arbiter::arbitrate() {
|
213 | // handle arbitration, DO NOT RE-ORDER these calls!
|
||
214 | 0c286af0 | Simon Schulz | |
215 | f150aab5 | Robert Haschke | // try to process all incoming requests
|
216 | override_by_emotion(); |
||
217 | 0c286af0 | Simon Schulz | |
218 | f150aab5 | Robert Haschke | // overwrite with mouth targets from outside:
|
219 | override_by_mouth(); |
||
220 | 0c286af0 | Simon Schulz | |
221 | f150aab5 | Robert Haschke | // lip animation
|
222 | override_by_utterance(); |
||
223 | 0c286af0 | Simon Schulz | |
224 | f150aab5 | Robert Haschke | // animations
|
225 | override_by_animation(); |
||
226 | 0c286af0 | Simon Schulz | |
227 | f150aab5 | Robert Haschke | // gaze target is set by requested_gaze_target
|
228 | override_by_gaze(); |
||
229 | 0c286af0 | Simon Schulz | } |
230 | |||
231 | f150aab5 | Robert Haschke | void Arbiter::override_by_utterance() {
|
232 | // fetch MouthState by utterance:
|
||
233 | if (!utterance->is_playing()) {
|
||
234 | // not playing -> return
|
||
235 | return;
|
||
236 | } |
||
237 | 0c286af0 | Simon Schulz | |
238 | f150aab5 | Robert Haschke | // fetch symbol
|
239 | string symbol = utterance->currently_active_phoneme();
|
||
240 | if ((symbol.empty()) || (symbol == "")) { |
||
241 | return;
|
||
242 | } |
||
243 | 0c286af0 | Simon Schulz | |
244 | f150aab5 | Robert Haschke | // fetch config
|
245 | mouth_config.init_by_symbol(symbol); |
||
246 | 0c286af0 | Simon Schulz | |
247 | f150aab5 | Robert Haschke | // overwrite mouth with offsets from utterance player
|
248 | mouth_config.apply_on_mouth_state(&mouth_state); |
||
249 | 0c286af0 | Simon Schulz | } |
250 | |||
251 | f150aab5 | Robert Haschke | void Arbiter::override_by_mouth() {
|
252 | // external mouth requests can overwrite the mouth targets!
|
||
253 | if ((requested_mouth_state.position_center + requested_mouth_state.position_left + requested_mouth_state.position_right) >
|
||
254 | 0.0) { |
||
255 | // values given -> override emotion data
|
||
256 | mouth_state = requested_mouth_state; |
||
257 | } |
||
258 | // requested_mouth_state.dump();
|
||
259 | // mouth_state.dump();
|
||
260 | 0c286af0 | Simon Schulz | } |
261 | |||
262 | f150aab5 | Robert Haschke | void Arbiter::override_by_gaze() {
|
263 | // copy from requested target:
|
||
264 | gaze_state.pan = requested_gaze_state.pan; |
||
265 | gaze_state.tilt = requested_gaze_state.tilt; |
||
266 | gaze_state.roll = requested_gaze_state.roll; |
||
267 | 0c286af0 | Simon Schulz | |
268 | f150aab5 | Robert Haschke | gaze_state.gaze_type = requested_gaze_state.gaze_type; |
269 | gaze_state.timestamp = requested_gaze_state.timestamp; |
||
270 | gaze_state.vergence = requested_gaze_state.vergence; |
||
271 | 0c286af0 | Simon Schulz | |
272 | f150aab5 | Robert Haschke | // add offset values from user:
|
273 | gaze_state.pan_offset += requested_gaze_state.pan_offset; |
||
274 | gaze_state.tilt_offset += requested_gaze_state.tilt_offset; |
||
275 | 0c286af0 | Simon Schulz | } |
276 | |||
277 | f150aab5 | Robert Haschke | void Arbiter::override_by_emotion() {
|
278 | if (emotion_target == NULL) { |
||
279 | return;
|
||
280 | } |
||
281 | |||
282 | // lock access
|
||
283 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock1(emotion_config_default_mutex);
|
284 | f150aab5 | Robert Haschke | |
285 | // did the current emotion time out?
|
||
286 | if (!emotion_target->is_active()) {
|
||
287 | // revert to default:
|
||
288 | emotion_target = &emotion_config_default; |
||
289 | // trigger soft fade
|
||
290 | gaze_state_animation_restart = true;
|
||
291 | } |
||
292 | |||
293 | // emotions will set the mouth state here (will be overwritten by speak)
|
||
294 | mouth_state = emotion_target->mouth_override; |
||
295 | // mouth_state = ...
|
||
296 | |||
297 | // emotions will add an offset to the gaze state:
|
||
298 | // pan,tilt,roll will be overwritten by override_by_gaze() we only keep the offset values from this!
|
||
299 | // FIXME: change to eyelid opening upper/lower once humotion supports it
|
||
300 | // eyebrow angles come from this
|
||
301 | humotion::GazeState gaze_state_target = emotion_target->gaze_override; |
||
302 | // apply a soft overblending to the new gaze target
|
||
303 | gaze_state = soft_overblend_gaze(gaze_state, gaze_state_target, emotion_target->overblend_time_ms); |
||
304 | |||
305 | // gaze_state.dump();
|
||
306 | |||
307 | // IMPORTANT: clear request from emotion target:
|
||
308 | emotion_target->gaze_override.eyeblink_request_left = 0;
|
||
309 | emotion_target->gaze_override.eyeblink_request_right = 0;
|
||
310 | efbcb710 | sschulz | } |
311 | |||
312 | f150aab5 | Robert Haschke | humotion::GazeState Arbiter::soft_overblend_gaze(humotion::GazeState gaze_now, humotion::GazeState gaze_target, |
313 | efbcb710 | sschulz | unsigned int overblend_time) { |
314 | f150aab5 | Robert Haschke | humotion::GazeState result = gaze_now; |
315 | |||
316 | result.eyeblink_request_left = 0;
|
||
317 | result.eyeblink_request_right = 0;
|
||
318 | |||
319 | if (gaze_state_animation_restart) {
|
||
320 | // new incoming target, set up soft fade:
|
||
321 | cout << "NEW EMOTION\n";
|
||
322 | gaze_state_old = gaze_state; |
||
323 | 90fa4b4e | Robert Haschke | gaze_state_end_time = std::chrono::steady_clock::now() + std::chrono::milliseconds(overblend_time); |
324 | f150aab5 | Robert Haschke | gaze_state_animation_restart = false;
|
325 | } |
||
326 | // do smooth overblend, all targets should be reached at gaze_state_end_time
|
||
327 | 90fa4b4e | Robert Haschke | double diff_ms =
|
328 | std::chrono::duration<double, std::milli>(gaze_state_end_time - std::chrono::steady_clock::now()).count();
|
||
329 | if (diff_ms < 0) { |
||
330 | f150aab5 | Robert Haschke | // animation is done, exit now
|
331 | return gaze_target;
|
||
332 | } |
||
333 | else {
|
||
334 | // do smooth animation
|
||
335 | double tp = 1.0 - diff_ms / static_cast<double>(overblend_time); |
||
336 | |||
337 | result.pan_offset = gaze_state_old.pan_offset + tp * (gaze_target.pan_offset - gaze_state_old.pan_offset); |
||
338 | result.tilt_offset = gaze_state_old.tilt_offset + tp * (gaze_target.tilt_offset - gaze_state_old.tilt_offset); |
||
339 | result.roll_offset = gaze_state_old.roll_offset + tp * (gaze_target.roll_offset - gaze_state_old.roll_offset); |
||
340 | |||
341 | result.eyelid_opening_lower = |
||
342 | gaze_state_old.eyelid_opening_lower + tp * (gaze_target.eyelid_opening_lower - gaze_state_old.eyelid_opening_lower); |
||
343 | result.eyelid_opening_upper = |
||
344 | gaze_state_old.eyelid_opening_upper + tp * (gaze_target.eyelid_opening_upper - gaze_state_old.eyelid_opening_upper); |
||
345 | |||
346 | result.eyebrow_left = gaze_target.eyebrow_left; |
||
347 | result.eyebrow_right = gaze_target.eyebrow_right; |
||
348 | } |
||
349 | |||
350 | return result;
|
||
351 | 0c286af0 | Simon Schulz | } |
352 | |||
353 | f150aab5 | Robert Haschke | void Arbiter::override_by_animation() {
|
354 | // lock access & iterate over vector:
|
||
355 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock_av(active_animation_vector_mutex);
|
356 | f150aab5 | Robert Haschke | active_animation_vector_t::iterator it; |
357 | for (it = active_animation_vector.begin(); it < active_animation_vector.end(); it++) {
|
||
358 | 482adb6d | Robert Haschke | std::shared_ptr<Animation> current_ani = *it; |
359 | f150aab5 | Robert Haschke | |
360 | // gaze_state.dump();
|
||
361 | current_ani->apply_on_gazestate(&gaze_state); |
||
362 | // gaze_state.dump();
|
||
363 | } |
||
364 | 0c286af0 | Simon Schulz | } |
365 | |||
366 | f150aab5 | Robert Haschke | humotion::GazeState Arbiter::get_gaze_state() { |
367 | // gaze_state.dump();
|
||
368 | return gaze_state;
|
||
369 | 0c286af0 | Simon Schulz | } |
370 | |||
371 | f150aab5 | Robert Haschke | humotion::MouthState Arbiter::get_mouth_state() { |
372 | return mouth_state;
|
||
373 | 0c286af0 | Simon Schulz | } |