hlrc / server / src / Arbiter.cpp @ 42084118
History | View | Annotate | Download (11.365 KB)
1 | 0c286af0 | Simon Schulz | /*
|
---|---|---|---|
2 | f150aab5 | Robert Haschke | * This file is part of hlrc_server
|
3 | *
|
||
4 | * Copyright(c) sschulz <AT> techfak.uni-bielefeld.de
|
||
5 | * http://opensource.cit-ec.de/projects/hlrc_server
|
||
6 | *
|
||
7 | * This file may be licensed under the terms of the
|
||
8 | * GNU General Public License Version 3 (the ``GPL''),
|
||
9 | * or (at your option) any later version.
|
||
10 | *
|
||
11 | * Software distributed under the License is distributed
|
||
12 | * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
|
||
13 | * express or implied. See the GPL for the specific language
|
||
14 | * governing rights and limitations.
|
||
15 | *
|
||
16 | * You should have received a copy of the GPL along with this
|
||
17 | * program. If not, go to http://www.gnu.org/licenses/gpl.html
|
||
18 | * or write to the Free Software Foundation, Inc.,
|
||
19 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||
20 | *
|
||
21 | * The development of this software was supported by the
|
||
22 | * Excellence Cluster EXC 277 Cognitive Interaction Technology.
|
||
23 | * The Excellence Cluster EXC 277 is a grant of the Deutsche
|
||
24 | * Forschungsgemeinschaft (DFG) in the context of the German
|
||
25 | * Excellence Initiative.
|
||
26 | *
|
||
27 | */
|
||
28 | 0c286af0 | Simon Schulz | |
29 | #include "Arbiter.h" |
||
30 | #include "AudioPlayerLibAO.h" |
||
31 | #include "AudioPlayerRSB.h" |
||
32 | #include <boost/algorithm/string/predicate.hpp> |
||
33 | 42084118 | Robert Haschke | #include <mutex> |
34 | #include <thread> |
||
35 | 90fa4b4e | Robert Haschke | #include <iostream> |
36 | 0c286af0 | Simon Schulz | |
37 | using namespace boost; |
||
38 | using namespace std; |
||
39 | |||
40 | f150aab5 | Robert Haschke | Arbiter::Arbiter(std::string audio_output) {
|
41 | // initialize audio player:
|
||
42 | if (iequals(audio_output, "none")) { |
||
43 | // allow none for no sound output
|
||
44 | audio_player = NULL;
|
||
45 | } |
||
46 | else if (iequals(audio_output.substr(0, 3), "rsb")) { |
||
47 | #ifdef RSB_SUPPORT
|
||
48 | audio_player = new AudioPlayerRSB(audio_output);
|
||
49 | #else
|
||
50 | printf("> ERROR: hlc is compiled without RSB support, RSB audio transport not available, defaulting to libao (default "
|
||
51 | "output!)\n");
|
||
52 | audio_player = new AudioPlayerLibAO(""); |
||
53 | #endif
|
||
54 | } |
||
55 | else {
|
||
56 | audio_player = new AudioPlayerLibAO(audio_output);
|
||
57 | } |
||
58 | |||
59 | // set default emotion:
|
||
60 | emotion_config_current.init_sad(); |
||
61 | emotion_config_current.set_duration(2000);
|
||
62 | emotion_config_default.init_neutral(); |
||
63 | |||
64 | gaze_state_animation_restart = true;
|
||
65 | emotion_target = &emotion_config_default; |
||
66 | |||
67 | utterance = boost::shared_ptr<Utterance>(new Utterance());
|
||
68 | 0c286af0 | Simon Schulz | } |
69 | |||
70 | f150aab5 | Robert Haschke | Arbiter::~Arbiter() { |
71 | 0c286af0 | Simon Schulz | } |
72 | |||
73 | f150aab5 | Robert Haschke | void Arbiter::set_default_emotion(EmotionState e) {
|
74 | // lock access
|
||
75 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(emotion_config_default_mutex);
|
76 | 0c286af0 | Simon Schulz | |
77 | f150aab5 | Robert Haschke | // update our config, ignore duration as this is the default emotion
|
78 | emotion_config_default.select_config(e.value); |
||
79 | emotion_target = &emotion_config_default; |
||
80 | efbcb710 | sschulz | |
81 | f150aab5 | Robert Haschke | gaze_state_animation_restart = true;
|
82 | 0c286af0 | Simon Schulz | |
83 | f150aab5 | Robert Haschke | printf("> stored default emotion\n");
|
84 | 0c286af0 | Simon Schulz | } |
85 | |||
86 | f150aab5 | Robert Haschke | void Arbiter::set_current_emotion(EmotionState e) {
|
87 | // lock access
|
||
88 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(emotion_config_current_mutex);
|
89 | 0c286af0 | Simon Schulz | |
90 | f150aab5 | Robert Haschke | // this will set a emotion override for a given time, after the timeout
|
91 | // we will return to the default emotion state:
|
||
92 | emotion_config_current.select_config(e.value); |
||
93 | emotion_config_current.set_duration(e.duration); |
||
94 | 0c286af0 | Simon Schulz | |
95 | f150aab5 | Robert Haschke | emotion_target = &emotion_config_current; |
96 | gaze_state_animation_restart = true;
|
||
97 | efbcb710 | sschulz | |
98 | f150aab5 | Robert Haschke | printf("> stored current emotion (val=%d, duration = %dms)\n", e.value, e.duration);
|
99 | 0c286af0 | Simon Schulz | } |
100 | |||
101 | f150aab5 | Robert Haschke | void Arbiter::set_mouth_config(MouthConfig m) {
|
102 | // lock access
|
||
103 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(mouth_config_override_mutex);
|
104 | 0c286af0 | Simon Schulz | |
105 | f150aab5 | Robert Haschke | mouth_config = m; |
106 | 0c286af0 | Simon Schulz | } |
107 | |||
108 | f150aab5 | Robert Haschke | void Arbiter::set_gaze_target(humotion::GazeState target) {
|
109 | // lock access
|
||
110 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(requested_gaze_state_mutex);
|
111 | 0c286af0 | Simon Schulz | |
112 | f150aab5 | Robert Haschke | requested_gaze_state = target; |
113 | 0c286af0 | Simon Schulz | } |
114 | |||
115 | f150aab5 | Robert Haschke | void Arbiter::set_mouth_target(humotion::MouthState target) {
|
116 | // lock access
|
||
117 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock(requested_mouth_state_mutex);
|
118 | 0c286af0 | Simon Schulz | |
119 | f150aab5 | Robert Haschke | requested_mouth_state = target; |
120 | 0c286af0 | Simon Schulz | } |
121 | |||
122 | //! note: this is blocking!
|
||
123 | f150aab5 | Robert Haschke | void Arbiter::play_animation(boost::shared_ptr<Animation> incoming_animation) {
|
124 | // incoming animation, check if this would conflict with any pending animations:
|
||
125 | // lock access & iterate over vector:
|
||
126 | 42084118 | Robert Haschke | std::unique_lock<std::mutex> lock_av(active_animation_vector_mutex); |
127 | f150aab5 | Robert Haschke | active_animation_vector_t::iterator it; |
128 | for (it = active_animation_vector.begin(); it < active_animation_vector.end(); it++) {
|
||
129 | boost::shared_ptr<Animation> current_ani = *it; |
||
130 | // check if the running animation collides with the incoming animation:
|
||
131 | if (current_ani->collides_with(incoming_animation.get())) {
|
||
132 | // this would fail, we can not play this animation right now!
|
||
133 | throw runtime_error("incoming animation collides with unfinished active animation"); |
||
134 | return;
|
||
135 | } |
||
136 | } |
||
137 | |||
138 | // ok, fine. no other animations are active that would collide with the new request
|
||
139 | // thus it is safe to enqueue this item:
|
||
140 | active_animation_vector.push_back(incoming_animation); |
||
141 | |||
142 | // unlock mutex
|
||
143 | lock_av.unlock(); |
||
144 | |||
145 | // start playback!
|
||
146 | incoming_animation->start(); |
||
147 | |||
148 | // block until ani was played back
|
||
149 | while (incoming_animation->is_active()) {
|
||
150 | // printf("A"); fflush(stdout);
|
||
151 | usleep(1 * 1000); // 1ms, save cpu cycles |
||
152 | } |
||
153 | |||
154 | // ok, it finished. we can safely remove it now:
|
||
155 | lock_av.lock(); |
||
156 | for (it = active_animation_vector.begin(); it < active_animation_vector.end();) {
|
||
157 | boost::shared_ptr<Animation> current_ani = *it; |
||
158 | if (*it == incoming_animation) {
|
||
159 | // printf(">match -> remove incoming ani again\n");
|
||
160 | it = active_animation_vector.erase(it); |
||
161 | } |
||
162 | else {
|
||
163 | ++it; |
||
164 | } |
||
165 | } |
||
166 | 0c286af0 | Simon Schulz | } |
167 | |||
168 | f150aab5 | Robert Haschke | void Arbiter::speak(
|
169 | boost::shared_ptr<Utterance> u) { //, ao_sample_format audio_format, char *audio_data, unsigned int audio_len){
|
||
170 | // lock audio playback as such:
|
||
171 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock_audio(audio_player_mutex);
|
172 | f150aab5 | Robert Haschke | |
173 | 42084118 | Robert Haschke | { |
174 | // lock utterance & store data:
|
||
175 | const std::lock_guard<std::mutex> lock(utterance_mutex);
|
||
176 | utterance = u; |
||
177 | } |
||
178 | f150aab5 | Robert Haschke | |
179 | // start audio playback, this function returns once we started playback
|
||
180 | if (audio_player == NULL) { |
||
181 | printf("> audio_player disabled, not speaking '%s'\n", u.get()->get_text().c_str());
|
||
182 | } |
||
183 | else {
|
||
184 | audio_player->play(utterance->get_audio_data()); |
||
185 | } |
||
186 | utterance->start_playback(); |
||
187 | |||
188 | // wait until audio playback was finished:
|
||
189 | if (audio_player != NULL) { |
||
190 | while (audio_player->is_playing()) {
|
||
191 | // save some cpu cycles:
|
||
192 | usleep(1 * 1000); // 1ms |
||
193 | } |
||
194 | } |
||
195 | |||
196 | // in case the audio output fails, we end up here before the utterance is finished.
|
||
197 | // so check now if the utterance finished as well:
|
||
198 | while (utterance->is_playing()) {
|
||
199 | // save some cpu cycles:
|
||
200 | usleep(1 * 1000); // 1ms |
||
201 | } |
||
202 | 0c286af0 | Simon Schulz | } |
203 | |||
204 | f150aab5 | Robert Haschke | bool Arbiter::speak_active() {
|
205 | if (audio_player == NULL) { |
||
206 | return false; |
||
207 | } |
||
208 | else {
|
||
209 | return audio_player->is_playing();
|
||
210 | } |
||
211 | 0c286af0 | Simon Schulz | } |
212 | |||
213 | f150aab5 | Robert Haschke | void Arbiter::arbitrate() {
|
214 | // handle arbitration, DO NOT RE-ORDER these calls!
|
||
215 | 0c286af0 | Simon Schulz | |
216 | f150aab5 | Robert Haschke | // try to process all incoming requests
|
217 | override_by_emotion(); |
||
218 | 0c286af0 | Simon Schulz | |
219 | f150aab5 | Robert Haschke | // overwrite with mouth targets from outside:
|
220 | override_by_mouth(); |
||
221 | 0c286af0 | Simon Schulz | |
222 | f150aab5 | Robert Haschke | // lip animation
|
223 | override_by_utterance(); |
||
224 | 0c286af0 | Simon Schulz | |
225 | f150aab5 | Robert Haschke | // animations
|
226 | override_by_animation(); |
||
227 | 0c286af0 | Simon Schulz | |
228 | f150aab5 | Robert Haschke | // gaze target is set by requested_gaze_target
|
229 | override_by_gaze(); |
||
230 | 0c286af0 | Simon Schulz | } |
231 | |||
232 | f150aab5 | Robert Haschke | void Arbiter::override_by_utterance() {
|
233 | // fetch MouthState by utterance:
|
||
234 | if (!utterance->is_playing()) {
|
||
235 | // not playing -> return
|
||
236 | return;
|
||
237 | } |
||
238 | 0c286af0 | Simon Schulz | |
239 | f150aab5 | Robert Haschke | // fetch symbol
|
240 | string symbol = utterance->currently_active_phoneme();
|
||
241 | if ((symbol.empty()) || (symbol == "")) { |
||
242 | return;
|
||
243 | } |
||
244 | 0c286af0 | Simon Schulz | |
245 | f150aab5 | Robert Haschke | // fetch config
|
246 | mouth_config.init_by_symbol(symbol); |
||
247 | 0c286af0 | Simon Schulz | |
248 | f150aab5 | Robert Haschke | // overwrite mouth with offsets from utterance player
|
249 | mouth_config.apply_on_mouth_state(&mouth_state); |
||
250 | 0c286af0 | Simon Schulz | } |
251 | |||
252 | f150aab5 | Robert Haschke | void Arbiter::override_by_mouth() {
|
253 | // external mouth requests can overwrite the mouth targets!
|
||
254 | if ((requested_mouth_state.position_center + requested_mouth_state.position_left + requested_mouth_state.position_right) >
|
||
255 | 0.0) { |
||
256 | // values given -> override emotion data
|
||
257 | mouth_state = requested_mouth_state; |
||
258 | } |
||
259 | // requested_mouth_state.dump();
|
||
260 | // mouth_state.dump();
|
||
261 | 0c286af0 | Simon Schulz | } |
262 | |||
263 | f150aab5 | Robert Haschke | void Arbiter::override_by_gaze() {
|
264 | // copy from requested target:
|
||
265 | gaze_state.pan = requested_gaze_state.pan; |
||
266 | gaze_state.tilt = requested_gaze_state.tilt; |
||
267 | gaze_state.roll = requested_gaze_state.roll; |
||
268 | 0c286af0 | Simon Schulz | |
269 | f150aab5 | Robert Haschke | gaze_state.gaze_type = requested_gaze_state.gaze_type; |
270 | gaze_state.timestamp = requested_gaze_state.timestamp; |
||
271 | gaze_state.vergence = requested_gaze_state.vergence; |
||
272 | 0c286af0 | Simon Schulz | |
273 | f150aab5 | Robert Haschke | // add offset values from user:
|
274 | gaze_state.pan_offset += requested_gaze_state.pan_offset; |
||
275 | gaze_state.tilt_offset += requested_gaze_state.tilt_offset; |
||
276 | 0c286af0 | Simon Schulz | } |
277 | |||
278 | f150aab5 | Robert Haschke | void Arbiter::override_by_emotion() {
|
279 | if (emotion_target == NULL) { |
||
280 | return;
|
||
281 | } |
||
282 | |||
283 | // lock access
|
||
284 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock1(emotion_config_default_mutex);
|
285 | f150aab5 | Robert Haschke | |
286 | // did the current emotion time out?
|
||
287 | if (!emotion_target->is_active()) {
|
||
288 | // revert to default:
|
||
289 | emotion_target = &emotion_config_default; |
||
290 | // trigger soft fade
|
||
291 | gaze_state_animation_restart = true;
|
||
292 | } |
||
293 | |||
294 | // emotions will set the mouth state here (will be overwritten by speak)
|
||
295 | mouth_state = emotion_target->mouth_override; |
||
296 | // mouth_state = ...
|
||
297 | |||
298 | // emotions will add an offset to the gaze state:
|
||
299 | // pan,tilt,roll will be overwritten by override_by_gaze() we only keep the offset values from this!
|
||
300 | // FIXME: change to eyelid opening upper/lower once humotion supports it
|
||
301 | // eyebrow angles come from this
|
||
302 | humotion::GazeState gaze_state_target = emotion_target->gaze_override; |
||
303 | // apply a soft overblending to the new gaze target
|
||
304 | gaze_state = soft_overblend_gaze(gaze_state, gaze_state_target, emotion_target->overblend_time_ms); |
||
305 | |||
306 | // gaze_state.dump();
|
||
307 | |||
308 | // IMPORTANT: clear request from emotion target:
|
||
309 | emotion_target->gaze_override.eyeblink_request_left = 0;
|
||
310 | emotion_target->gaze_override.eyeblink_request_right = 0;
|
||
311 | efbcb710 | sschulz | } |
312 | |||
313 | f150aab5 | Robert Haschke | humotion::GazeState Arbiter::soft_overblend_gaze(humotion::GazeState gaze_now, humotion::GazeState gaze_target, |
314 | efbcb710 | sschulz | unsigned int overblend_time) { |
315 | f150aab5 | Robert Haschke | humotion::GazeState result = gaze_now; |
316 | |||
317 | result.eyeblink_request_left = 0;
|
||
318 | result.eyeblink_request_right = 0;
|
||
319 | |||
320 | if (gaze_state_animation_restart) {
|
||
321 | // new incoming target, set up soft fade:
|
||
322 | cout << "NEW EMOTION\n";
|
||
323 | gaze_state_old = gaze_state; |
||
324 | 90fa4b4e | Robert Haschke | gaze_state_end_time = std::chrono::steady_clock::now() + std::chrono::milliseconds(overblend_time); |
325 | f150aab5 | Robert Haschke | gaze_state_animation_restart = false;
|
326 | } |
||
327 | // do smooth overblend, all targets should be reached at gaze_state_end_time
|
||
328 | 90fa4b4e | Robert Haschke | double diff_ms =
|
329 | std::chrono::duration<double, std::milli>(gaze_state_end_time - std::chrono::steady_clock::now()).count();
|
||
330 | if (diff_ms < 0) { |
||
331 | f150aab5 | Robert Haschke | // animation is done, exit now
|
332 | return gaze_target;
|
||
333 | } |
||
334 | else {
|
||
335 | // do smooth animation
|
||
336 | double tp = 1.0 - diff_ms / static_cast<double>(overblend_time); |
||
337 | |||
338 | result.pan_offset = gaze_state_old.pan_offset + tp * (gaze_target.pan_offset - gaze_state_old.pan_offset); |
||
339 | result.tilt_offset = gaze_state_old.tilt_offset + tp * (gaze_target.tilt_offset - gaze_state_old.tilt_offset); |
||
340 | result.roll_offset = gaze_state_old.roll_offset + tp * (gaze_target.roll_offset - gaze_state_old.roll_offset); |
||
341 | |||
342 | result.eyelid_opening_lower = |
||
343 | gaze_state_old.eyelid_opening_lower + tp * (gaze_target.eyelid_opening_lower - gaze_state_old.eyelid_opening_lower); |
||
344 | result.eyelid_opening_upper = |
||
345 | gaze_state_old.eyelid_opening_upper + tp * (gaze_target.eyelid_opening_upper - gaze_state_old.eyelid_opening_upper); |
||
346 | |||
347 | result.eyebrow_left = gaze_target.eyebrow_left; |
||
348 | result.eyebrow_right = gaze_target.eyebrow_right; |
||
349 | } |
||
350 | |||
351 | return result;
|
||
352 | 0c286af0 | Simon Schulz | } |
353 | |||
354 | f150aab5 | Robert Haschke | void Arbiter::override_by_animation() {
|
355 | // lock access & iterate over vector:
|
||
356 | 42084118 | Robert Haschke | const std::lock_guard<std::mutex> lock_av(active_animation_vector_mutex);
|
357 | f150aab5 | Robert Haschke | active_animation_vector_t::iterator it; |
358 | for (it = active_animation_vector.begin(); it < active_animation_vector.end(); it++) {
|
||
359 | boost::shared_ptr<Animation> current_ani = *it; |
||
360 | |||
361 | // gaze_state.dump();
|
||
362 | current_ani->apply_on_gazestate(&gaze_state); |
||
363 | // gaze_state.dump();
|
||
364 | } |
||
365 | 0c286af0 | Simon Schulz | } |
366 | |||
367 | f150aab5 | Robert Haschke | humotion::GazeState Arbiter::get_gaze_state() { |
368 | // gaze_state.dump();
|
||
369 | return gaze_state;
|
||
370 | 0c286af0 | Simon Schulz | } |
371 | |||
372 | f150aab5 | Robert Haschke | humotion::MouthState Arbiter::get_mouth_state() { |
373 | return mouth_state;
|
||
374 | 0c286af0 | Simon Schulz | } |