Statistics
| Branch: | Tag: | Revision:

hlrc / server / src / Arbiter.cpp @ 888a909b

History | View | Annotate | Download (12.188 KB)

1
/*
2
* This file is part of hlrc_server
3
*
4
* Copyright(c) sschulz <AT> techfak.uni-bielefeld.de
5
* http://opensource.cit-ec.de/projects/hlrc_server
6
*
7
* This file may be licensed under the terms of the
8
* GNU General Public License Version 3 (the ``GPL''),
9
* or (at your option) any later version.
10
*
11
* Software distributed under the License is distributed
12
* on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
13
* express or implied. See the GPL for the specific language
14
* governing rights and limitations.
15
*
16
* You should have received a copy of the GPL along with this
17
* program. If not, go to http://www.gnu.org/licenses/gpl.html
18
* or write to the Free Software Foundation, Inc.,
19
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20
*
21
* The development of this software was supported by the
22
* Excellence Cluster EXC 277 Cognitive Interaction Technology.
23
* The Excellence Cluster EXC 277 is a grant of the Deutsche
24
* Forschungsgemeinschaft (DFG) in the context of the German
25
* Excellence Initiative.
26
*
27
*/
28

    
29
#include "Arbiter.h"
30
#include "AudioPlayerLibAO.h"
31
#include "AudioPlayerRSB.h"
32
#include <boost/algorithm/string/predicate.hpp>
33
#include <mutex>
34
#include <thread>
35
#include <iostream>
36

    
37
using namespace boost;
38
using namespace std;
39

    
40
Arbiter::Arbiter(std::string audio_output){
41
    //initialize audio player:
42
    if (iequals(audio_output, "none")){
43
        //allow none for no sound output
44
        audio_player = NULL;
45
    }else if (iequals(audio_output.substr(0,3), "rsb")){
46
        #ifdef RSB_SUPPORT
47
            audio_player = new AudioPlayerRSB(audio_output);
48
        #else
49
            printf("> ERROR: hlc is compiled without RSB support, RSB audio transport not available, defaulting to libao (default output!)\n");
50
            audio_player = new AudioPlayerLibAO("");
51
        #endif
52
    }else{
53
        audio_player = new AudioPlayerLibAO(audio_output);
54
    }
55

    
56
    //set default emotion:
57
    emotion_config_current.init_sad();
58
    emotion_config_current.set_duration(2000);
59
    emotion_config_default.init_neutral();
60

    
61
    gaze_state_animation_restart = true;
62
    emotion_target = &emotion_config_default;
63

    
64
    utterance = std::shared_ptr<Utterance>(new Utterance());
65
}
66

    
67
Arbiter::~Arbiter(){
68
}
69

    
70
void Arbiter::set_default_emotion(EmotionState e){
71
    //lock access
72
    const std::lock_guard<std::mutex> lock(emotion_config_default_mutex);
73

    
74
    //update our config, ignore duration as this is the default emotion
75
    emotion_config_default.select_config(e.value);
76
    emotion_target = &emotion_config_default;
77

    
78
    gaze_state_animation_restart = true;
79

    
80
    printf("> stored default emotion\n");
81
}
82

    
83
void Arbiter::set_current_emotion(EmotionState e){
84
    //lock access
85
    const std::lock_guard<std::mutex> lock(emotion_config_current_mutex);
86

    
87
    //this will set a emotion override for a given time, after the timeout
88
    //we will return to the default emotion state:
89
    emotion_config_current.select_config(e.value);
90
    emotion_config_current.set_duration(e.duration);
91

    
92
    emotion_target = &emotion_config_current;
93
    gaze_state_animation_restart = true;
94

    
95
    printf("> stored current emotion (val=%d, duration = %dms)\n",e.value, e.duration);
96
}
97

    
98
void Arbiter::set_mouth_config(MouthConfig m){
99
    //lock access
100
    const std::lock_guard<std::mutex> lock(mouth_config_override_mutex);
101

    
102
    mouth_config = m;
103
}
104

    
105
void Arbiter::set_gaze_target(humotion::GazeState target){
106
    //lock access
107
    const std::lock_guard<std::mutex> lock(requested_gaze_state_mutex);
108

    
109
    requested_gaze_state = target;
110
}
111

    
112
void Arbiter::set_mouth_target(humotion::MouthState target){
113
    //lock access
114
    const std::lock_guard<std::mutex> lock(requested_mouth_state_mutex);
115

    
116
    requested_mouth_state = target;
117
}
118

    
119
//! note: this is blocking!
120
void Arbiter::play_animation(std::shared_ptr<Animation> incoming_animation){
121
    //incoming animation, check if this would conflict with any pending animations:
122
    //lock access & iterate over vector:
123
    std::unique_lock<std::mutex> lock_av(active_animation_vector_mutex);
124
    active_animation_vector_t::iterator it;
125
    for(it = active_animation_vector.begin(); it<active_animation_vector.end(); it++){
126
        std::shared_ptr<Animation> current_ani = *it;
127
        //check if the running animation collides with the incoming animation:
128
        if (current_ani->collides_with(incoming_animation.get())){
129
            //this would fail, we can not play this animation right now!
130
            throw runtime_error("incoming animation collides with unfinished active animation");
131
            return;
132
        }
133
    }
134

    
135
    //ok, fine. no other animations are active that would collide with the new request
136
    //thus it is safe to enqueue this item:
137
    active_animation_vector.push_back(incoming_animation);
138

    
139
    //unlock mutex
140
    lock_av.unlock();
141

    
142
    //start playback!
143
    incoming_animation->start();
144

    
145
    //block until ani was played back
146
    while(incoming_animation->is_active()){
147
        std::this_thread::sleep_for(std::chrono::milliseconds(1));//save cpu cycles
148
    }
149

    
150
    //ok, it finished. we can safely remove it now:
151
    lock_av.lock();
152
    for(it = active_animation_vector.begin(); it<active_animation_vector.end();){
153
        std::shared_ptr<Animation> current_ani = *it;
154
        if (*it == incoming_animation){
155
            //printf(">match -> remove incoming ani again\n");
156
            it = active_animation_vector.erase(it);
157
          }else{
158
            ++it;
159
          }
160
    }
161
}
162

    
163
void Arbiter::speak(std::shared_ptr<Utterance> u){ //, ao_sample_format audio_format, char *audio_data, unsigned int audio_len){
164
    //lock audio playback as such:
165
    const std::lock_guard<std::mutex> lock_audio(audio_player_mutex);
166

    
167
    {
168
        //lock utterance & store data:
169
        const std::lock_guard<std::mutex> lock(utterance_mutex);
170
        utterance = u;
171
    }
172

    
173
    //start audio playback, this function returns once we started playback
174
    if (audio_player == NULL){
175
        printf("> audio_player disabled, not speaking '%s'\n", u.get()->get_text().c_str());
176
    }else{
177
        audio_player->play(utterance->get_audio_data());
178
    }
179
    utterance->start_playback();
180

    
181
    //wait until audio playback was finished:
182
    if (audio_player != NULL){
183
        while(audio_player->is_playing()){
184
            //save some cpu cycles:
185
            std::this_thread::sleep_for(std::chrono::milliseconds(1)); //1ms
186
        }
187
    }
188

    
189
    //in case the audio output fails, we end up here before the utterance is finished.
190
    //so check now if the utterance finished as well:
191
    while (utterance->is_playing()){
192
        //save some cpu cycles:
193
        std::this_thread::sleep_for(std::chrono::milliseconds(1));
194
    }
195

    
196
}
197

    
198
bool Arbiter::speak_active(){
199
    if (audio_player == NULL){
200
        return false;
201
    }else{
202
        return audio_player->is_playing();
203
    }
204
}
205

    
206
void Arbiter::arbitrate(){
207
    //handle arbitration, DO NOT RE-ORDER these calls!
208

    
209
    //try to process all incoming requests
210
    override_by_emotion();
211

    
212
    //overwrite with mouth targets from outside:
213
    override_by_mouth();
214

    
215
    //lip animation
216
    override_by_utterance();
217

    
218
    //animations
219
    override_by_animation();
220

    
221
    //gaze target is set by requested_gaze_target
222
    override_by_gaze();
223
}
224

    
225
void Arbiter::override_by_utterance(){
226
    //fetch MouthState by utterance:
227
    if (!utterance->is_playing()){
228
        //not playing -> return
229
        return;
230
    }
231

    
232
    //fetch symbol
233
    string symbol = utterance->currently_active_phoneme();
234
    if ((symbol.empty()) || (symbol == "")){
235
        return;
236
    }
237

    
238
    //fetch config
239
    mouth_config.init_by_symbol(symbol);
240

    
241
    //overwrite mouth with offsets from utterance player
242
    mouth_config.apply_on_mouth_state(&mouth_state);
243
}
244

    
245
void Arbiter::override_by_mouth(){
246
    //external mouth requests can overwrite the mouth targets!
247
    if ((requested_mouth_state.position_center + requested_mouth_state.position_left + requested_mouth_state.position_right) > 0.0){
248
        //values given -> override emotion data
249
        mouth_state = requested_mouth_state;
250
    }
251
    //requested_mouth_state.dump();
252
    //mouth_state.dump();
253
}
254

    
255
void Arbiter::override_by_gaze(){
256
    //copy from requested target:
257
    gaze_state.pan   = requested_gaze_state.pan;
258
    gaze_state.tilt  = requested_gaze_state.tilt;
259
    gaze_state.roll  = requested_gaze_state.roll;
260

    
261
    gaze_state.gaze_type = requested_gaze_state.gaze_type;
262
    gaze_state.timestamp = requested_gaze_state.timestamp;
263
    gaze_state.vergence  = requested_gaze_state.vergence;
264

    
265
    //add offset values from user:
266
    gaze_state.pan_offset  += requested_gaze_state.pan_offset;
267
    gaze_state.tilt_offset += requested_gaze_state.tilt_offset;
268
}
269

    
270
void Arbiter::override_by_emotion(){
271
    if (emotion_target == NULL) {
272
        return;
273
    }
274

    
275
    //lock access
276
    const std::lock_guard<std::mutex> lock1(emotion_config_default_mutex);
277

    
278

    
279
    // did the current emotion time out?
280
    if (!emotion_target->is_active()){
281
        // revert to default:
282
        emotion_target = &emotion_config_default;
283
        // trigger soft fade
284
        gaze_state_animation_restart = true;
285
    }
286

    
287
    //emotions will set the mouth state here (will be overwritten by speak)
288
    mouth_state = emotion_target->mouth_override;
289
    //mouth_state = ...
290

    
291
    //emotions will add an offset to the gaze state:
292
    //pan,tilt,roll will be overwritten by override_by_gaze() we only keep the offset values from this!
293
    //FIXME: change to eyelid opening  upper/lower once humotion supports it
294
    //eyebrow angles come from this
295
    humotion::GazeState gaze_state_target = emotion_target->gaze_override;
296
    // apply a soft overblending to the new gaze target
297
    gaze_state = soft_overblend_gaze(gaze_state, gaze_state_target, emotion_target->overblend_time_ms);
298

    
299
    //gaze_state.dump();
300

    
301
    //IMPORTANT: clear request from emotion target:
302
    emotion_target->gaze_override.eyeblink_request_left = 0;
303
    emotion_target->gaze_override.eyeblink_request_right = 0;
304
}
305

    
306
humotion::GazeState Arbiter::soft_overblend_gaze(humotion::GazeState gaze_now,
307
                                                 humotion::GazeState gaze_target,
308
                                                 unsigned int overblend_time) {
309
    humotion::GazeState result = gaze_now;
310

    
311
    result.eyeblink_request_left = 0;
312
    result.eyeblink_request_right = 0;
313

    
314
    if (gaze_state_animation_restart) {
315
        // new incoming target, set up soft fade:
316
        cout << "NEW EMOTION\n";
317
        gaze_state_old = gaze_state;
318
        gaze_state_end_time = std::chrono::steady_clock::now() + std::chrono::milliseconds(overblend_time);
319
        gaze_state_animation_restart = false;
320
    }
321
    // do smooth overblend, all targets should be reached at gaze_state_end_time
322
    double diff_ms = std::chrono::duration<double, std::milli>(gaze_state_end_time - std::chrono::steady_clock::now()).count();
323
    if (diff_ms < 0) {
324
        // animation is done, exit now
325
        return gaze_target;
326
    } else {
327
        // do smooth animation
328
        double tp = 1.0 - diff_ms / static_cast<double>(overblend_time);
329

    
330
        result.pan_offset = gaze_state_old.pan_offset + tp * (gaze_target.pan_offset - gaze_state_old.pan_offset);
331
        result.tilt_offset = gaze_state_old.tilt_offset + tp * (gaze_target.tilt_offset - gaze_state_old.tilt_offset);
332
        result.roll_offset = gaze_state_old.roll_offset + tp * (gaze_target.roll_offset - gaze_state_old.roll_offset);
333

    
334
        result.eyelid_opening_lower = gaze_state_old.eyelid_opening_lower + tp * (gaze_target.eyelid_opening_lower - gaze_state_old.eyelid_opening_lower);
335
        result.eyelid_opening_upper = gaze_state_old.eyelid_opening_upper + tp * (gaze_target.eyelid_opening_upper - gaze_state_old.eyelid_opening_upper);
336

    
337
        result.eyebrow_left = gaze_target.eyebrow_left;
338
        result.eyebrow_right = gaze_target.eyebrow_right;
339

    
340
    }
341

    
342
    return result;
343
}
344

    
345

    
346
void Arbiter::override_by_animation(){
347
    //lock access & iterate over vector:
348
    const std::lock_guard<std::mutex> lock_av(active_animation_vector_mutex);
349
    active_animation_vector_t::iterator it;
350
    for(it = active_animation_vector.begin(); it<active_animation_vector.end(); it++){
351
        std::shared_ptr<Animation> current_ani = *it;
352

    
353
        //gaze_state.dump();
354
        current_ani->apply_on_gazestate(&gaze_state);
355
        //gaze_state.dump();
356
    }
357
}
358

    
359
humotion::GazeState Arbiter::get_gaze_state(){
360
    //gaze_state.dump();
361
    return gaze_state;
362
}
363

    
364
humotion::MouthState Arbiter::get_mouth_state(){
365
    return mouth_state;
366
}