hlrc / server / src / Arbiter.cpp @ ff49ed31
History | View | Annotate | Download (11.363 KB)
1 | 0c286af0 | Simon Schulz | /*
|
---|---|---|---|
2 | f150aab5 | Robert Haschke | * This file is part of hlrc_server
|
3 | *
|
||
4 | * Copyright(c) sschulz <AT> techfak.uni-bielefeld.de
|
||
5 | * http://opensource.cit-ec.de/projects/hlrc_server
|
||
6 | *
|
||
7 | * This file may be licensed under the terms of the
|
||
8 | * GNU General Public License Version 3 (the ``GPL''),
|
||
9 | * or (at your option) any later version.
|
||
10 | *
|
||
11 | * Software distributed under the License is distributed
|
||
12 | * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
|
||
13 | * express or implied. See the GPL for the specific language
|
||
14 | * governing rights and limitations.
|
||
15 | *
|
||
16 | * You should have received a copy of the GPL along with this
|
||
17 | * program. If not, go to http://www.gnu.org/licenses/gpl.html
|
||
18 | * or write to the Free Software Foundation, Inc.,
|
||
19 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||
20 | *
|
||
21 | * The development of this software was supported by the
|
||
22 | * Excellence Cluster EXC 277 Cognitive Interaction Technology.
|
||
23 | * The Excellence Cluster EXC 277 is a grant of the Deutsche
|
||
24 | * Forschungsgemeinschaft (DFG) in the context of the German
|
||
25 | * Excellence Initiative.
|
||
26 | *
|
||
27 | */
|
||
28 | 0c286af0 | Simon Schulz | |
29 | #include "Arbiter.h" |
||
30 | #include "AudioPlayerLibAO.h" |
||
31 | #include "AudioPlayerRSB.h" |
||
32 | #include <boost/algorithm/string/predicate.hpp> |
||
33 | 42084118 | Robert Haschke | #include <mutex> |
34 | #include <thread> |
||
35 | 90fa4b4e | Robert Haschke | #include <iostream> |
36 | 0c286af0 | Simon Schulz | |
37 | using namespace boost; |
||
38 | using namespace std; |
||
39 | |||
40 | f150aab5 | Robert Haschke | Arbiter::Arbiter(std::string audio_output) {
|
41 | // initialize audio player:
|
||
42 | if (iequals(audio_output, "none")) { |
||
43 | // allow none for no sound output
|
||
44 | audio_player = NULL;
|
||
45 | } |
||
46 | ff49ed31 | robocamp | else if (iequals(audio_output.substr(0, 3), "rsb")) { |
47 | printf("> ERROR: hlc is compiled without RSB support, RSB audio transport not available, defaulting to libao (default "
|
||
48 | "output!)\n");
|
||
49 | audio_player = new AudioPlayerLibAO(""); |
||
50 | } |
||
51 | f150aab5 | Robert Haschke | else {
|
52 | audio_player = new AudioPlayerLibAO(audio_output);
|
||
53 | } |
||
54 | |||
55 | // set default emotion:
|
||
56 | emotion_config_current.init_sad(); |
||
57 | emotion_config_current.set_duration(2000);
|
||
58 | emotion_config_default.init_neutral(); |
||
59 | |||
60 | gaze_state_animation_restart = true;
|
||
61 | emotion_target = &emotion_config_default; |
||
62 | |||
63 | 482adb6d | Robert Haschke | utterance = std::shared_ptr<Utterance>(new Utterance());
|
64 | 0c286af0 | Simon Schulz | } |
65 | |||
66 | f150aab5 | Robert Haschke | Arbiter::~Arbiter() { |
67 | 0c286af0 | Simon Schulz | } |
68 | |||
69 | f150aab5 | Robert Haschke | void Arbiter::set_default_emotion(EmotionState e) {
|
70 | // lock access
|
||
71 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(emotion_config_default_mutex);
|
72 | 0c286af0 | Simon Schulz | |
73 | f150aab5 | Robert Haschke | // update our config, ignore duration as this is the default emotion
|
74 | emotion_config_default.select_config(e.value); |
||
75 | emotion_target = &emotion_config_default; |
||
76 | efbcb710 | sschulz | |
77 | f150aab5 | Robert Haschke | gaze_state_animation_restart = true;
|
78 | 0c286af0 | Simon Schulz | |
79 | f150aab5 | Robert Haschke | printf("> stored default emotion\n");
|
80 | 0c286af0 | Simon Schulz | } |
81 | |||
82 | f150aab5 | Robert Haschke | void Arbiter::set_current_emotion(EmotionState e) {
|
83 | // lock access
|
||
84 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(emotion_config_current_mutex);
|
85 | 0c286af0 | Simon Schulz | |
86 | f150aab5 | Robert Haschke | // this will set a emotion override for a given time, after the timeout
|
87 | // we will return to the default emotion state:
|
||
88 | emotion_config_current.select_config(e.value); |
||
89 | emotion_config_current.set_duration(e.duration); |
||
90 | 0c286af0 | Simon Schulz | |
91 | f150aab5 | Robert Haschke | emotion_target = &emotion_config_current; |
92 | gaze_state_animation_restart = true;
|
||
93 | efbcb710 | sschulz | |
94 | f150aab5 | Robert Haschke | printf("> stored current emotion (val=%d, duration = %dms)\n", e.value, e.duration);
|
95 | 0c286af0 | Simon Schulz | } |
96 | |||
97 | f150aab5 | Robert Haschke | void Arbiter::set_mouth_config(MouthConfig m) {
|
98 | // lock access
|
||
99 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(mouth_config_override_mutex);
|
100 | 0c286af0 | Simon Schulz | |
101 | f150aab5 | Robert Haschke | mouth_config = m; |
102 | 0c286af0 | Simon Schulz | } |
103 | |||
104 | f150aab5 | Robert Haschke | void Arbiter::set_gaze_target(humotion::GazeState target) {
|
105 | // lock access
|
||
106 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(requested_gaze_state_mutex);
|
107 | 0c286af0 | Simon Schulz | |
108 | f150aab5 | Robert Haschke | requested_gaze_state = target; |
109 | 0c286af0 | Simon Schulz | } |
110 | |||
111 | f150aab5 | Robert Haschke | void Arbiter::set_mouth_target(humotion::MouthState target) {
|
112 | // lock access
|
||
113 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(requested_mouth_state_mutex);
|
114 | 0c286af0 | Simon Schulz | |
115 | f150aab5 | Robert Haschke | requested_mouth_state = target; |
116 | 0c286af0 | Simon Schulz | } |
117 | |||
118 | //! note: this is blocking!
|
||
119 | 482adb6d | Robert Haschke | void Arbiter::play_animation(std::shared_ptr<Animation> incoming_animation) {
|
120 | f150aab5 | Robert Haschke | // incoming animation, check if this would conflict with any pending animations:
|
121 | // lock access & iterate over vector:
|
||
122 | 42084118 | Robert Haschke | std::unique_lock<std::mutex> lock_av(active_animation_vector_mutex); |
123 | f150aab5 | Robert Haschke | active_animation_vector_t::iterator it; |
124 | for (it = active_animation_vector.begin(); it < active_animation_vector.end(); it++) {
|
||
125 | 482adb6d | Robert Haschke | std::shared_ptr<Animation> current_ani = *it; |
126 | f150aab5 | Robert Haschke | // check if the running animation collides with the incoming animation:
|
127 | if (current_ani->collides_with(incoming_animation.get())) {
|
||
128 | // this would fail, we can not play this animation right now!
|
||
129 | throw runtime_error("incoming animation collides with unfinished active animation"); |
||
130 | return;
|
||
131 | } |
||
132 | } |
||
133 | |||
134 | // ok, fine. no other animations are active that would collide with the new request
|
||
135 | // thus it is safe to enqueue this item:
|
||
136 | active_animation_vector.push_back(incoming_animation); |
||
137 | |||
138 | // unlock mutex
|
||
139 | lock_av.unlock(); |
||
140 | |||
141 | // start playback!
|
||
142 | incoming_animation->start(); |
||
143 | |||
144 | // block until ani was played back
|
||
145 | while (incoming_animation->is_active()) {
|
||
146 | e8a50634 | Robert Haschke | std::this_thread::sleep_for(std::chrono::milliseconds(1)); // save cpu cycles |
147 | f150aab5 | Robert Haschke | } |
148 | |||
149 | // ok, it finished. we can safely remove it now:
|
||
150 | lock_av.lock(); |
||
151 | for (it = active_animation_vector.begin(); it < active_animation_vector.end();) {
|
||
152 | 482adb6d | Robert Haschke | std::shared_ptr<Animation> current_ani = *it; |
153 | f150aab5 | Robert Haschke | if (*it == incoming_animation) {
|
154 | // printf(">match -> remove incoming ani again\n");
|
||
155 | it = active_animation_vector.erase(it); |
||
156 | } |
||
157 | else {
|
||
158 | ++it; |
||
159 | } |
||
160 | } |
||
161 | 0c286af0 | Simon Schulz | } |
162 | |||
163 | f150aab5 | Robert Haschke | void Arbiter::speak(
|
164 | 482adb6d | Robert Haschke | std::shared_ptr<Utterance> u) { //, ao_sample_format audio_format, char *audio_data, unsigned int audio_len){
|
165 | f150aab5 | Robert Haschke | // lock audio playback as such:
|
166 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock_audio(audio_player_mutex);
|
167 | f150aab5 | Robert Haschke | |
168 | 42084118 | Robert Haschke | { |
169 | // lock utterance & store data:
|
||
170 | const std::lock_guard<std::mutex> lock(utterance_mutex);
|
||
171 | utterance = u; |
||
172 | } |
||
173 | f150aab5 | Robert Haschke | |
174 | // start audio playback, this function returns once we started playback
|
||
175 | if (audio_player == NULL) { |
||
176 | printf("> audio_player disabled, not speaking '%s'\n", u.get()->get_text().c_str());
|
||
177 | } |
||
178 | else {
|
||
179 | audio_player->play(utterance->get_audio_data()); |
||
180 | } |
||
181 | utterance->start_playback(); |
||
182 | |||
183 | // wait until audio playback was finished:
|
||
184 | if (audio_player != NULL) { |
||
185 | while (audio_player->is_playing()) {
|
||
186 | // save some cpu cycles:
|
||
187 | e8a50634 | Robert Haschke | std::this_thread::sleep_for(std::chrono::milliseconds(1)); // 1ms |
188 | f150aab5 | Robert Haschke | } |
189 | } |
||
190 | |||
191 | // in case the audio output fails, we end up here before the utterance is finished.
|
||
192 | // so check now if the utterance finished as well:
|
||
193 | while (utterance->is_playing()) {
|
||
194 | // save some cpu cycles:
|
||
195 | e8a50634 | Robert Haschke | std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
196 | f150aab5 | Robert Haschke | } |
197 | 0c286af0 | Simon Schulz | } |
198 | |||
199 | f150aab5 | Robert Haschke | bool Arbiter::speak_active() {
|
200 | if (audio_player == NULL) { |
||
201 | return false; |
||
202 | } |
||
203 | else {
|
||
204 | return audio_player->is_playing();
|
||
205 | } |
||
206 | 0c286af0 | Simon Schulz | } |
207 | |||
208 | f150aab5 | Robert Haschke | void Arbiter::arbitrate() {
|
209 | // handle arbitration, DO NOT RE-ORDER these calls!
|
||
210 | 0c286af0 | Simon Schulz | |
211 | f150aab5 | Robert Haschke | // try to process all incoming requests
|
212 | override_by_emotion(); |
||
213 | 0c286af0 | Simon Schulz | |
214 | f150aab5 | Robert Haschke | // overwrite with mouth targets from outside:
|
215 | override_by_mouth(); |
||
216 | 0c286af0 | Simon Schulz | |
217 | f150aab5 | Robert Haschke | // lip animation
|
218 | override_by_utterance(); |
||
219 | 0c286af0 | Simon Schulz | |
220 | f150aab5 | Robert Haschke | // animations
|
221 | override_by_animation(); |
||
222 | 0c286af0 | Simon Schulz | |
223 | f150aab5 | Robert Haschke | // gaze target is set by requested_gaze_target
|
224 | override_by_gaze(); |
||
225 | 0c286af0 | Simon Schulz | } |
226 | |||
227 | f150aab5 | Robert Haschke | void Arbiter::override_by_utterance() {
|
228 | // fetch MouthState by utterance:
|
||
229 | if (!utterance->is_playing()) {
|
||
230 | // not playing -> return
|
||
231 | return;
|
||
232 | } |
||
233 | 0c286af0 | Simon Schulz | |
234 | f150aab5 | Robert Haschke | // fetch symbol
|
235 | string symbol = utterance->currently_active_phoneme();
|
||
236 | if ((symbol.empty()) || (symbol == "")) { |
||
237 | return;
|
||
238 | } |
||
239 | 0c286af0 | Simon Schulz | |
240 | f150aab5 | Robert Haschke | // fetch config
|
241 | mouth_config.init_by_symbol(symbol); |
||
242 | 0c286af0 | Simon Schulz | |
243 | f150aab5 | Robert Haschke | // overwrite mouth with offsets from utterance player
|
244 | mouth_config.apply_on_mouth_state(&mouth_state); |
||
245 | 0c286af0 | Simon Schulz | } |
246 | |||
247 | f150aab5 | Robert Haschke | void Arbiter::override_by_mouth() {
|
248 | // external mouth requests can overwrite the mouth targets!
|
||
249 | if ((requested_mouth_state.position_center + requested_mouth_state.position_left + requested_mouth_state.position_right) >
|
||
250 | 0.0) { |
||
251 | // values given -> override emotion data
|
||
252 | mouth_state = requested_mouth_state; |
||
253 | } |
||
254 | // requested_mouth_state.dump();
|
||
255 | // mouth_state.dump();
|
||
256 | 0c286af0 | Simon Schulz | } |
257 | |||
258 | f150aab5 | Robert Haschke | void Arbiter::override_by_gaze() {
|
259 | // copy from requested target:
|
||
260 | gaze_state.pan = requested_gaze_state.pan; |
||
261 | gaze_state.tilt = requested_gaze_state.tilt; |
||
262 | gaze_state.roll = requested_gaze_state.roll; |
||
263 | 0c286af0 | Simon Schulz | |
264 | f150aab5 | Robert Haschke | gaze_state.gaze_type = requested_gaze_state.gaze_type; |
265 | gaze_state.timestamp = requested_gaze_state.timestamp; |
||
266 | gaze_state.vergence = requested_gaze_state.vergence; |
||
267 | 0c286af0 | Simon Schulz | |
268 | f150aab5 | Robert Haschke | // add offset values from user:
|
269 | gaze_state.pan_offset += requested_gaze_state.pan_offset; |
||
270 | gaze_state.tilt_offset += requested_gaze_state.tilt_offset; |
||
271 | 0c286af0 | Simon Schulz | } |
272 | |||
273 | f150aab5 | Robert Haschke | void Arbiter::override_by_emotion() {
|
274 | if (emotion_target == NULL) { |
||
275 | return;
|
||
276 | } |
||
277 | |||
278 | // lock access
|
||
279 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock1(emotion_config_default_mutex);
|
280 | f150aab5 | Robert Haschke | |
281 | // did the current emotion time out?
|
||
282 | if (!emotion_target->is_active()) {
|
||
283 | // revert to default:
|
||
284 | emotion_target = &emotion_config_default; |
||
285 | // trigger soft fade
|
||
286 | gaze_state_animation_restart = true;
|
||
287 | } |
||
288 | |||
289 | // emotions will set the mouth state here (will be overwritten by speak)
|
||
290 | mouth_state = emotion_target->mouth_override; |
||
291 | // mouth_state = ...
|
||
292 | |||
293 | // emotions will add an offset to the gaze state:
|
||
294 | // pan,tilt,roll will be overwritten by override_by_gaze() we only keep the offset values from this!
|
||
295 | // FIXME: change to eyelid opening upper/lower once humotion supports it
|
||
296 | // eyebrow angles come from this
|
||
297 | humotion::GazeState gaze_state_target = emotion_target->gaze_override; |
||
298 | // apply a soft overblending to the new gaze target
|
||
299 | gaze_state = soft_overblend_gaze(gaze_state, gaze_state_target, emotion_target->overblend_time_ms); |
||
300 | |||
301 | // gaze_state.dump();
|
||
302 | |||
303 | // IMPORTANT: clear request from emotion target:
|
||
304 | emotion_target->gaze_override.eyeblink_request_left = 0;
|
||
305 | emotion_target->gaze_override.eyeblink_request_right = 0;
|
||
306 | efbcb710 | sschulz | } |
307 | |||
308 | f150aab5 | Robert Haschke | humotion::GazeState Arbiter::soft_overblend_gaze(humotion::GazeState gaze_now, humotion::GazeState gaze_target, |
309 | efbcb710 | sschulz | unsigned int overblend_time) { |
310 | f150aab5 | Robert Haschke | humotion::GazeState result = gaze_now; |
311 | |||
312 | result.eyeblink_request_left = 0;
|
||
313 | result.eyeblink_request_right = 0;
|
||
314 | |||
315 | if (gaze_state_animation_restart) {
|
||
316 | // new incoming target, set up soft fade:
|
||
317 | cout << "NEW EMOTION\n";
|
||
318 | gaze_state_old = gaze_state; |
||
319 | 90fa4b4e | Robert Haschke | gaze_state_end_time = std::chrono::steady_clock::now() + std::chrono::milliseconds(overblend_time); |
320 | f150aab5 | Robert Haschke | gaze_state_animation_restart = false;
|
321 | } |
||
322 | // do smooth overblend, all targets should be reached at gaze_state_end_time
|
||
323 | 90fa4b4e | Robert Haschke | double diff_ms =
|
324 | std::chrono::duration<double, std::milli>(gaze_state_end_time - std::chrono::steady_clock::now()).count();
|
||
325 | if (diff_ms < 0) { |
||
326 | f150aab5 | Robert Haschke | // animation is done, exit now
|
327 | return gaze_target;
|
||
328 | } |
||
329 | else {
|
||
330 | // do smooth animation
|
||
331 | double tp = 1.0 - diff_ms / static_cast<double>(overblend_time); |
||
332 | |||
333 | result.pan_offset = gaze_state_old.pan_offset + tp * (gaze_target.pan_offset - gaze_state_old.pan_offset); |
||
334 | result.tilt_offset = gaze_state_old.tilt_offset + tp * (gaze_target.tilt_offset - gaze_state_old.tilt_offset); |
||
335 | result.roll_offset = gaze_state_old.roll_offset + tp * (gaze_target.roll_offset - gaze_state_old.roll_offset); |
||
336 | |||
337 | result.eyelid_opening_lower = |
||
338 | gaze_state_old.eyelid_opening_lower + tp * (gaze_target.eyelid_opening_lower - gaze_state_old.eyelid_opening_lower); |
||
339 | result.eyelid_opening_upper = |
||
340 | gaze_state_old.eyelid_opening_upper + tp * (gaze_target.eyelid_opening_upper - gaze_state_old.eyelid_opening_upper); |
||
341 | |||
342 | result.eyebrow_left = gaze_target.eyebrow_left; |
||
343 | result.eyebrow_right = gaze_target.eyebrow_right; |
||
344 | } |
||
345 | |||
346 | return result;
|
||
347 | 0c286af0 | Simon Schulz | } |
348 | |||
349 | f150aab5 | Robert Haschke | void Arbiter::override_by_animation() {
|
350 | // lock access & iterate over vector:
|
||
351 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock_av(active_animation_vector_mutex);
|
352 | f150aab5 | Robert Haschke | active_animation_vector_t::iterator it; |
353 | for (it = active_animation_vector.begin(); it < active_animation_vector.end(); it++) {
|
||
354 | 482adb6d | Robert Haschke | std::shared_ptr<Animation> current_ani = *it; |
355 | f150aab5 | Robert Haschke | |
356 | // gaze_state.dump();
|
||
357 | current_ani->apply_on_gazestate(&gaze_state); |
||
358 | // gaze_state.dump();
|
||
359 | } |
||
360 | 0c286af0 | Simon Schulz | } |
361 | |||
362 | f150aab5 | Robert Haschke | humotion::GazeState Arbiter::get_gaze_state() { |
363 | // gaze_state.dump();
|
||
364 | return gaze_state;
|
||
365 | 0c286af0 | Simon Schulz | } |
366 | |||
367 | f150aab5 | Robert Haschke | humotion::MouthState Arbiter::get_mouth_state() { |
368 | return mouth_state;
|
||
369 | 0c286af0 | Simon Schulz | } |