hlrc

/*

* This file is part of hlrc_server

*

* Copyright(c) sschulz <AT> techfak.uni-bielefeld.de

* http://opensource.cit-ec.de/projects/hlrc_server

*

* This file may be licensed under the terms of the

* GNU General Public License Version 3 (the ``GPL''),

* or (at your option) any later version.

*

* Software distributed under the License is distributed

* on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either

* express or implied. See the GPL for the specific language

* governing rights and limitations.

*

* You should have received a copy of the GPL along with this

* program. If not, go to http://www.gnu.org/licenses/gpl.html

* or write to the Free Software Foundation, Inc.,

* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

*

* The development of this software was supported by the

* Excellence Cluster EXC 277 Cognitive Interaction Technology.

* The Excellence Cluster EXC 277 is a grant of the Deutsche

* Forschungsgemeinschaft (DFG) in the context of the German

* Excellence Initiative.

*

*/

#include "Arbiter.h"

#include "AudioPlayerLibAO.h"

#include "AudioPlayerRSB.h"

#include <boost/algorithm/string/predicate.hpp>

#include <mutex>

#include <thread>

#include <iostream>

using namespace boost;

using namespace std;

Arbiter::Arbiter(std::string audio_output){

    //initialize audio player:

    if (iequals(audio_output, "none")){

        //allow none for no sound output

        audio_player = NULL;

    }else if (iequals(audio_output.substr(0,3), "rsb")){

        #ifdef RSB_SUPPORT

            audio_player = new AudioPlayerRSB(audio_output);

        #else

            printf("> ERROR: hlc is compiled without RSB support, RSB audio transport not available, defaulting to libao (default output!)\n");

            audio_player = new AudioPlayerLibAO("");

        #endif

    }else{

        audio_player = new AudioPlayerLibAO(audio_output);

    }

    //set default emotion:

    emotion_config_current.init_sad();

    emotion_config_current.set_duration(2000);

    emotion_config_default.init_neutral();

    gaze_state_animation_restart = true;

    emotion_target = &emotion_config_default;

    utterance = boost::shared_ptr<Utterance>(new Utterance());

}

Arbiter::~Arbiter(){

}

void Arbiter::set_default_emotion(EmotionState e){

    //lock access

    const std::lock_guard<std::mutex> lock(emotion_config_default_mutex);

    //update our config, ignore duration as this is the default emotion

    emotion_config_default.select_config(e.value);

    emotion_target = &emotion_config_default;

    gaze_state_animation_restart = true;

    printf("> stored default emotion\n");

}

void Arbiter::set_current_emotion(EmotionState e){

    //lock access

    const std::lock_guard<std::mutex> lock(emotion_config_current_mutex);

    //this will set a emotion override for a given time, after the timeout

    //we will return to the default emotion state:

    emotion_config_current.select_config(e.value);

    emotion_config_current.set_duration(e.duration);

    emotion_target = &emotion_config_current;

    gaze_state_animation_restart = true;

    printf("> stored current emotion (val=%d, duration = %dms)\n",e.value, e.duration);

}

void Arbiter::set_mouth_config(MouthConfig m){

    //lock access

    const std::lock_guard<std::mutex> lock(mouth_config_override_mutex);

    mouth_config = m;

}

void Arbiter::set_gaze_target(humotion::GazeState target){

    //lock access

    const std::lock_guard<std::mutex> lock(requested_gaze_state_mutex);

    requested_gaze_state = target;

}

void Arbiter::set_mouth_target(humotion::MouthState target){

    //lock access

    const std::lock_guard<std::mutex> lock(requested_mouth_state_mutex);

    requested_mouth_state = target;

}

//! note: this is blocking!

void Arbiter::play_animation(boost::shared_ptr<Animation> incoming_animation){

    //incoming animation, check if this would conflict with any pending animations:

    //lock access & iterate over vector:

    std::unique_lock<std::mutex> lock_av(active_animation_vector_mutex);

    active_animation_vector_t::iterator it;

    for(it = active_animation_vector.begin(); it<active_animation_vector.end(); it++){

        boost::shared_ptr<Animation> current_ani = *it;

        //check if the running animation collides with the incoming animation:

        if (current_ani->collides_with(incoming_animation.get())){

            //this would fail, we can not play this animation right now!

            throw runtime_error("incoming animation collides with unfinished active animation");

            return;

        }

    }

    //ok, fine. no other animations are active that would collide with the new request

    //thus it is safe to enqueue this item:

    active_animation_vector.push_back(incoming_animation);

    //unlock mutex

    lock_av.unlock();

    //start playback!

    incoming_animation->start();

    //block until ani was played back

    while(incoming_animation->is_active()){

        std::this_thread::sleep_for(std::chrono::milliseconds(1));//save cpu cycles

    }

    //ok, it finished. we can safely remove it now:

    lock_av.lock();

    for(it = active_animation_vector.begin(); it<active_animation_vector.end();){

        boost::shared_ptr<Animation> current_ani = *it;

        if (*it == incoming_animation){

            //printf(">match -> remove incoming ani again\n");

            it = active_animation_vector.erase(it);

          }else{

            ++it;

          }

    }

}

void Arbiter::speak(boost::shared_ptr<Utterance> u){ //, ao_sample_format audio_format, char *audio_data, unsigned int audio_len){

    //lock audio playback as such:

    const std::lock_guard<std::mutex> lock_audio(audio_player_mutex);

    {

        //lock utterance & store data:

        const std::lock_guard<std::mutex> lock(utterance_mutex);

        utterance = u;

    }

    //start audio playback, this function returns once we started playback

    if (audio_player == NULL){

        printf("> audio_player disabled, not speaking '%s'\n", u.get()->get_text().c_str());

    }else{

        audio_player->play(utterance->get_audio_data());

    }

    utterance->start_playback();

    //wait until audio playback was finished:

    if (audio_player != NULL){

        while(audio_player->is_playing()){

            //save some cpu cycles:

            std::this_thread::sleep_for(std::chrono::milliseconds(1)); //1ms

        }

    }

    //in case the audio output fails, we end up here before the utterance is finished.

    //so check now if the utterance finished as well:

    while (utterance->is_playing()){

        //save some cpu cycles:

        std::this_thread::sleep_for(std::chrono::milliseconds(1));

    }

}

bool Arbiter::speak_active(){

    if (audio_player == NULL){

        return false;

    }else{

        return audio_player->is_playing();

    }

}

void Arbiter::arbitrate(){

    //handle arbitration, DO NOT RE-ORDER these calls!

    //try to process all incoming requests

    override_by_emotion();

    //overwrite with mouth targets from outside:

    override_by_mouth();

    //lip animation

    override_by_utterance();

    //animations

    override_by_animation();

    //gaze target is set by requested_gaze_target

    override_by_gaze();

}

void Arbiter::override_by_utterance(){

    //fetch MouthState by utterance:

    if (!utterance->is_playing()){

        //not playing -> return

        return;

    }

    //fetch symbol

    string symbol = utterance->currently_active_phoneme();

    if ((symbol.empty()) || (symbol == "")){

        return;

    }

    //fetch config

    mouth_config.init_by_symbol(symbol);

    //overwrite mouth with offsets from utterance player

    mouth_config.apply_on_mouth_state(&mouth_state);

}

void Arbiter::override_by_mouth(){

    //external mouth requests can overwrite the mouth targets!

    if ((requested_mouth_state.position_center + requested_mouth_state.position_left + requested_mouth_state.position_right) > 0.0){

        //values given -> override emotion data

        mouth_state = requested_mouth_state;

    }

    //requested_mouth_state.dump();

    //mouth_state.dump();

}

void Arbiter::override_by_gaze(){

    //copy from requested target:

    gaze_state.pan   = requested_gaze_state.pan;

    gaze_state.tilt  = requested_gaze_state.tilt;

    gaze_state.roll  = requested_gaze_state.roll;

    gaze_state.gaze_type = requested_gaze_state.gaze_type;

    gaze_state.timestamp = requested_gaze_state.timestamp;

    gaze_state.vergence  = requested_gaze_state.vergence;

    //add offset values from user:

    gaze_state.pan_offset  += requested_gaze_state.pan_offset;

    gaze_state.tilt_offset += requested_gaze_state.tilt_offset;

}

void Arbiter::override_by_emotion(){

    if (emotion_target == NULL) {

        return;

    }

    //lock access

    const std::lock_guard<std::mutex> lock1(emotion_config_default_mutex);

    // did the current emotion time out?

    if (!emotion_target->is_active()){

        // revert to default:

        emotion_target = &emotion_config_default;

        // trigger soft fade

        gaze_state_animation_restart = true;

    }

    //emotions will set the mouth state here (will be overwritten by speak)

    mouth_state = emotion_target->mouth_override;

    //mouth_state = ...

    //emotions will add an offset to the gaze state:

    //pan,tilt,roll will be overwritten by override_by_gaze() we only keep the offset values from this!

    //FIXME: change to eyelid opening  upper/lower once humotion supports it

    //eyebrow angles come from this

    humotion::GazeState gaze_state_target = emotion_target->gaze_override;

    // apply a soft overblending to the new gaze target

    gaze_state = soft_overblend_gaze(gaze_state, gaze_state_target, emotion_target->overblend_time_ms);

    //gaze_state.dump();

    //IMPORTANT: clear request from emotion target:

    emotion_target->gaze_override.eyeblink_request_left = 0;

    emotion_target->gaze_override.eyeblink_request_right = 0;

}

humotion::GazeState Arbiter::soft_overblend_gaze(humotion::GazeState gaze_now,

                                                 humotion::GazeState gaze_target,

                                                 unsigned int overblend_time) {

    humotion::GazeState result = gaze_now;

    result.eyeblink_request_left = 0;

    result.eyeblink_request_right = 0;

    if (gaze_state_animation_restart) {

        // new incoming target, set up soft fade:

        cout << "NEW EMOTION\n";

        gaze_state_old = gaze_state;

        gaze_state_end_time = std::chrono::steady_clock::now() + std::chrono::milliseconds(overblend_time);

        gaze_state_animation_restart = false;

    }

    // do smooth overblend, all targets should be reached at gaze_state_end_time

    double diff_ms = std::chrono::duration<double, std::milli>(gaze_state_end_time - std::chrono::steady_clock::now()).count();

    if (diff_ms < 0) {

        // animation is done, exit now

        return gaze_target;

    } else {

        // do smooth animation

        double tp = 1.0 - diff_ms / static_cast<double>(overblend_time);

        result.pan_offset = gaze_state_old.pan_offset + tp * (gaze_target.pan_offset - gaze_state_old.pan_offset);

        result.tilt_offset = gaze_state_old.tilt_offset + tp * (gaze_target.tilt_offset - gaze_state_old.tilt_offset);

        result.roll_offset = gaze_state_old.roll_offset + tp * (gaze_target.roll_offset - gaze_state_old.roll_offset);

        result.eyelid_opening_lower = gaze_state_old.eyelid_opening_lower + tp * (gaze_target.eyelid_opening_lower - gaze_state_old.eyelid_opening_lower);

        result.eyelid_opening_upper = gaze_state_old.eyelid_opening_upper + tp * (gaze_target.eyelid_opening_upper - gaze_state_old.eyelid_opening_upper);

        result.eyebrow_left = gaze_target.eyebrow_left;

        result.eyebrow_right = gaze_target.eyebrow_right;

    }

    return result;

}

void Arbiter::override_by_animation(){

    //lock access & iterate over vector:

    const std::lock_guard<std::mutex> lock_av(active_animation_vector_mutex);

    active_animation_vector_t::iterator it;

    for(it = active_animation_vector.begin(); it<active_animation_vector.end(); it++){

        boost::shared_ptr<Animation> current_ani = *it;

        //gaze_state.dump();

        current_ani->apply_on_gazestate(&gaze_state);

        //gaze_state.dump();

    }

}

humotion::GazeState Arbiter::get_gaze_state(){

    //gaze_state.dump();

    return gaze_state;

}

humotion::MouthState Arbiter::get_mouth_state(){

    return mouth_state;

}