Statistics
| Branch: | Tag: | Revision:

hlrc / server / src / Arbiter.cpp @ d8feb404

History | View | Annotate | Download (11.326 KB)

1
/*
2
 * This file is part of hlrc_server
3
 *
4
 * Copyright(c) sschulz <AT> techfak.uni-bielefeld.de
5
 * http://opensource.cit-ec.de/projects/hlrc_server
6
 *
7
 * This file may be licensed under the terms of the
8
 * GNU General Public License Version 3 (the ``GPL''),
9
 * or (at your option) any later version.
10
 *
11
 * Software distributed under the License is distributed
12
 * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
13
 * express or implied. See the GPL for the specific language
14
 * governing rights and limitations.
15
 *
16
 * You should have received a copy of the GPL along with this
17
 * program. If not, go to http://www.gnu.org/licenses/gpl.html
18
 * or write to the Free Software Foundation, Inc.,
19
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20
 *
21
 * The development of this software was supported by the
22
 * Excellence Cluster EXC 277 Cognitive Interaction Technology.
23
 * The Excellence Cluster EXC 277 is a grant of the Deutsche
24
 * Forschungsgemeinschaft (DFG) in the context of the German
25
 * Excellence Initiative.
26
 *
27
 */
28

    
29
#include "Arbiter.h"
30
#include "AudioPlayerLibAO.h"
31
#include <boost/algorithm/string/predicate.hpp>
32
#include <mutex>
33
#include <thread>
34
#include <iostream>
35

    
36
using namespace boost;
37
using namespace std;
38

    
39
Arbiter::Arbiter(std::string audio_output) {
40
        // initialize audio player:
41
        if (iequals(audio_output, "none")) {
42
                // allow none for no sound output
43
                audio_player = NULL;
44
        }
45
   else if (iequals(audio_output.substr(0, 3), "rsb")) {
46
      printf("> ERROR: hlrc does not support RSB, RSB audio transport not available, defaulting to libao (default "
47
             "output!)\n");
48
      audio_player = new AudioPlayerLibAO("");
49
   }
50
        else {
51
                audio_player = new AudioPlayerLibAO(audio_output);
52
        }
53

    
54
        // set default emotion:
55
        emotion_config_current.init_sad();
56
        emotion_config_current.set_duration(2000);
57
        emotion_config_default.init_neutral();
58

    
59
        gaze_state_animation_restart = true;
60
        emotion_target = &emotion_config_default;
61

    
62
        utterance = std::shared_ptr<Utterance>(new Utterance());
63
}
64

    
65
Arbiter::~Arbiter() {
66
}
67

    
68
void Arbiter::set_default_emotion(EmotionState e) {
69
        // lock access
70
        const std::lock_guard<std::mutex> lock(emotion_config_default_mutex);
71

    
72
        // update our config, ignore duration as this is the default emotion
73
        emotion_config_default.select_config(e.value);
74
        emotion_target = &emotion_config_default;
75

    
76
        gaze_state_animation_restart = true;
77

    
78
        printf("> stored default emotion\n");
79
}
80

    
81
void Arbiter::set_current_emotion(EmotionState e) {
82
        // lock access
83
        const std::lock_guard<std::mutex> lock(emotion_config_current_mutex);
84

    
85
        // this will set a emotion override for a given time, after the timeout
86
        // we will return to the default emotion state:
87
        emotion_config_current.select_config(e.value);
88
        emotion_config_current.set_duration(e.duration);
89

    
90
        emotion_target = &emotion_config_current;
91
        gaze_state_animation_restart = true;
92

    
93
        printf("> stored current emotion (val=%d, duration = %dms)\n", e.value, e.duration);
94
}
95

    
96
void Arbiter::set_mouth_config(MouthConfig m) {
97
        // lock access
98
        const std::lock_guard<std::mutex> lock(mouth_config_override_mutex);
99

    
100
        mouth_config = m;
101
}
102

    
103
void Arbiter::set_gaze_target(humotion::GazeState target) {
104
        // lock access
105
        const std::lock_guard<std::mutex> lock(requested_gaze_state_mutex);
106

    
107
        requested_gaze_state = target;
108
}
109

    
110
void Arbiter::set_mouth_target(humotion::MouthState target) {
111
        // lock access
112
        const std::lock_guard<std::mutex> lock(requested_mouth_state_mutex);
113

    
114
        requested_mouth_state = target;
115
}
116

    
117
//! note: this is blocking!
118
void Arbiter::play_animation(std::shared_ptr<Animation> incoming_animation) {
119
        // incoming animation, check if this would conflict with any pending animations:
120
        // lock access & iterate over vector:
121
        std::unique_lock<std::mutex> lock_av(active_animation_vector_mutex);
122
        active_animation_vector_t::iterator it;
123
        for (it = active_animation_vector.begin(); it < active_animation_vector.end(); it++) {
124
                std::shared_ptr<Animation> current_ani = *it;
125
                // check if the running animation collides with the incoming animation:
126
                if (current_ani->collides_with(incoming_animation.get())) {
127
                        // this would fail, we can not play this animation right now!
128
                        throw runtime_error("incoming animation collides with unfinished active animation");
129
                        return;
130
                }
131
        }
132

    
133
        // ok, fine. no other animations are active that would collide with the new request
134
        // thus it is safe to enqueue this item:
135
        active_animation_vector.push_back(incoming_animation);
136

    
137
        // unlock mutex
138
        lock_av.unlock();
139

    
140
        // start playback!
141
        incoming_animation->start();
142

    
143
        // block until ani was played back
144
        while (incoming_animation->is_active()) {
145
                std::this_thread::sleep_for(std::chrono::milliseconds(1)); // save cpu cycles
146
        }
147

    
148
        // ok, it finished. we can safely remove it now:
149
        lock_av.lock();
150
        for (it = active_animation_vector.begin(); it < active_animation_vector.end();) {
151
                std::shared_ptr<Animation> current_ani = *it;
152
                if (*it == incoming_animation) {
153
                        // printf(">match -> remove incoming ani again\n");
154
                        it = active_animation_vector.erase(it);
155
                }
156
                else {
157
                        ++it;
158
                }
159
        }
160
}
161

    
162
void Arbiter::speak(
163
   std::shared_ptr<Utterance> u) { //, ao_sample_format audio_format, char *audio_data, unsigned int audio_len){
164
        // lock audio playback as such:
165
        const std::lock_guard<std::mutex> lock_audio(audio_player_mutex);
166

    
167
        {
168
                // lock utterance & store data:
169
                const std::lock_guard<std::mutex> lock(utterance_mutex);
170
                utterance = u;
171
        }
172

    
173
        // start audio playback, this function returns once we started playback
174
        if (audio_player == NULL) {
175
                printf("> audio_player disabled, not speaking '%s'\n", u.get()->get_text().c_str());
176
        }
177
        else {
178
                audio_player->play(utterance->get_audio_data());
179
        }
180
        utterance->start_playback();
181

    
182
        // wait until audio playback was finished:
183
        if (audio_player != NULL) {
184
                while (audio_player->is_playing()) {
185
                        // save some cpu cycles:
186
                        std::this_thread::sleep_for(std::chrono::milliseconds(1)); // 1ms
187
                }
188
        }
189

    
190
        // in case the audio output fails, we end up here before the utterance is finished.
191
        // so check now if the utterance finished as well:
192
        while (utterance->is_playing()) {
193
                // save some cpu cycles:
194
                std::this_thread::sleep_for(std::chrono::milliseconds(1));
195
        }
196
}
197

    
198
bool Arbiter::speak_active() {
199
        if (audio_player == NULL) {
200
                return false;
201
        }
202
        else {
203
                return audio_player->is_playing();
204
        }
205
}
206

    
207
void Arbiter::arbitrate() {
208
        // handle arbitration, DO NOT RE-ORDER these calls!
209

    
210
        // try to process all incoming requests
211
        override_by_emotion();
212

    
213
        // overwrite with mouth targets from outside:
214
        override_by_mouth();
215

    
216
        // lip animation
217
        override_by_utterance();
218

    
219
        // animations
220
        override_by_animation();
221

    
222
        // gaze target is set by requested_gaze_target
223
        override_by_gaze();
224
}
225

    
226
void Arbiter::override_by_utterance() {
227
        // fetch MouthState by utterance:
228
        if (!utterance->is_playing()) {
229
                // not playing -> return
230
                return;
231
        }
232

    
233
        // fetch symbol
234
        string symbol = utterance->currently_active_phoneme();
235
        if ((symbol.empty()) || (symbol == "")) {
236
                return;
237
        }
238

    
239
        // fetch config
240
        mouth_config.init_by_symbol(symbol);
241

    
242
        // overwrite mouth with offsets from utterance player
243
        mouth_config.apply_on_mouth_state(&mouth_state);
244
}
245

    
246
void Arbiter::override_by_mouth() {
247
        // external mouth requests can overwrite the mouth targets!
248
        if ((requested_mouth_state.position_center + requested_mouth_state.position_left + requested_mouth_state.position_right) >
249
            0.0) {
250
                // values given -> override emotion data
251
                mouth_state = requested_mouth_state;
252
        }
253
        // requested_mouth_state.dump();
254
        // mouth_state.dump();
255
}
256

    
257
void Arbiter::override_by_gaze() {
258
        // copy from requested target:
259
        gaze_state.pan = requested_gaze_state.pan;
260
        gaze_state.tilt = requested_gaze_state.tilt;
261
        gaze_state.roll = requested_gaze_state.roll;
262

    
263
        gaze_state.gaze_type = requested_gaze_state.gaze_type;
264
        gaze_state.timestamp = requested_gaze_state.timestamp;
265
        gaze_state.vergence = requested_gaze_state.vergence;
266

    
267
        // add offset values from user:
268
        gaze_state.pan_offset += requested_gaze_state.pan_offset;
269
        gaze_state.tilt_offset += requested_gaze_state.tilt_offset;
270
}
271

    
272
void Arbiter::override_by_emotion() {
273
        if (emotion_target == NULL) {
274
                return;
275
        }
276

    
277
        // lock access
278
        const std::lock_guard<std::mutex> lock1(emotion_config_default_mutex);
279

    
280
        // did the current emotion time out?
281
        if (!emotion_target->is_active()) {
282
                // revert to default:
283
                emotion_target = &emotion_config_default;
284
                // trigger soft fade
285
                gaze_state_animation_restart = true;
286
        }
287

    
288
        // emotions will set the mouth state here (will be overwritten by speak)
289
        mouth_state = emotion_target->mouth_override;
290
        // mouth_state = ...
291

    
292
        // emotions will add an offset to the gaze state:
293
        // pan,tilt,roll will be overwritten by override_by_gaze() we only keep the offset values from this!
294
        // FIXME: change to eyelid opening  upper/lower once humotion supports it
295
        // eyebrow angles come from this
296
        humotion::GazeState gaze_state_target = emotion_target->gaze_override;
297
        // apply a soft overblending to the new gaze target
298
        gaze_state = soft_overblend_gaze(gaze_state, gaze_state_target, emotion_target->overblend_time_ms);
299

    
300
        // gaze_state.dump();
301

    
302
        // IMPORTANT: clear request from emotion target:
303
        emotion_target->gaze_override.eyeblink_request_left = 0;
304
        emotion_target->gaze_override.eyeblink_request_right = 0;
305
}
306

    
307
humotion::GazeState Arbiter::soft_overblend_gaze(humotion::GazeState gaze_now, humotion::GazeState gaze_target,
308
                                                 unsigned int overblend_time) {
309
        humotion::GazeState result = gaze_now;
310

    
311
        result.eyeblink_request_left = 0;
312
        result.eyeblink_request_right = 0;
313

    
314
        if (gaze_state_animation_restart) {
315
                // new incoming target, set up soft fade:
316
                cout << "NEW EMOTION\n";
317
                gaze_state_old = gaze_state;
318
                gaze_state_end_time = std::chrono::steady_clock::now() + std::chrono::milliseconds(overblend_time);
319
                gaze_state_animation_restart = false;
320
        }
321
        // do smooth overblend, all targets should be reached at gaze_state_end_time
322
        double diff_ms =
323
           std::chrono::duration<double, std::milli>(gaze_state_end_time - std::chrono::steady_clock::now()).count();
324
        if (diff_ms < 0) {
325
                // animation is done, exit now
326
                return gaze_target;
327
        }
328
        else {
329
                // do smooth animation
330
                double tp = 1.0 - diff_ms / static_cast<double>(overblend_time);
331

    
332
                result.pan_offset = gaze_state_old.pan_offset + tp * (gaze_target.pan_offset - gaze_state_old.pan_offset);
333
                result.tilt_offset = gaze_state_old.tilt_offset + tp * (gaze_target.tilt_offset - gaze_state_old.tilt_offset);
334
                result.roll_offset = gaze_state_old.roll_offset + tp * (gaze_target.roll_offset - gaze_state_old.roll_offset);
335

    
336
                result.eyelid_opening_lower =
337
                   gaze_state_old.eyelid_opening_lower + tp * (gaze_target.eyelid_opening_lower - gaze_state_old.eyelid_opening_lower);
338
                result.eyelid_opening_upper =
339
                   gaze_state_old.eyelid_opening_upper + tp * (gaze_target.eyelid_opening_upper - gaze_state_old.eyelid_opening_upper);
340

    
341
                result.eyebrow_left = gaze_target.eyebrow_left;
342
                result.eyebrow_right = gaze_target.eyebrow_right;
343
        }
344

    
345
        return result;
346
}
347

    
348
void Arbiter::override_by_animation() {
349
        // lock access & iterate over vector:
350
        const std::lock_guard<std::mutex> lock_av(active_animation_vector_mutex);
351
        active_animation_vector_t::iterator it;
352
        for (it = active_animation_vector.begin(); it < active_animation_vector.end(); it++) {
353
                std::shared_ptr<Animation> current_ani = *it;
354

    
355
                // gaze_state.dump();
356
                current_ani->apply_on_gazestate(&gaze_state);
357
                // gaze_state.dump();
358
        }
359
}
360

    
361
humotion::GazeState Arbiter::get_gaze_state() {
362
        // gaze_state.dump();
363
        return gaze_state;
364
}
365

    
366
humotion::MouthState Arbiter::get_mouth_state() {
367
        return mouth_state;
368
}