hlrc

/*

 * This file is part of hlrc_server

 *

 * Copyright(c) sschulz <AT> techfak.uni-bielefeld.de

 * http://opensource.cit-ec.de/projects/hlrc_server

 *

 * This file may be licensed under the terms of the

 * GNU General Public License Version 3 (the ``GPL''),

 * or (at your option) any later version.

 *

 * Software distributed under the License is distributed

 * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either

 * express or implied. See the GPL for the specific language

 * governing rights and limitations.

 *

 * You should have received a copy of the GPL along with this

 * program. If not, go to http://www.gnu.org/licenses/gpl.html

 * or write to the Free Software Foundation, Inc.,

 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

 *

 * The development of this software was supported by the

 * Excellence Cluster EXC 277 Cognitive Interaction Technology.

 * The Excellence Cluster EXC 277 is a grant of the Deutsche

 * Forschungsgemeinschaft (DFG) in the context of the German

 * Excellence Initiative.

 *

 */

#include "Arbiter.h"

#include "AudioPlayerLibAO.h"

#include "AudioPlayerRSB.h"

#include <boost/algorithm/string/predicate.hpp>

#include <mutex>

#include <thread>

#include <iostream>

using namespace boost;

using namespace std;

Arbiter::Arbiter(std::string audio_output) {

        // initialize audio player:

        if (iequals(audio_output, "none")) {

                // allow none for no sound output

                audio_player = NULL;

        }

        else if (iequals(audio_output.substr(0, 3), "rsb")) {

#ifdef RSB_SUPPORT

                audio_player = new AudioPlayerRSB(audio_output);

#else

                printf("> ERROR: hlc is compiled without RSB support, RSB audio transport not available, defaulting to libao (default "

                       "output!)\n");

                audio_player = new AudioPlayerLibAO("");

#endif

        }

        else {

                audio_player = new AudioPlayerLibAO(audio_output);

        }

        // set default emotion:

        emotion_config_current.init_sad();

        emotion_config_current.set_duration(2000);

        emotion_config_default.init_neutral();

        gaze_state_animation_restart = true;

        emotion_target = &emotion_config_default;

        utterance = boost::shared_ptr<Utterance>(new Utterance());

}

Arbiter::~Arbiter() {

}

void Arbiter::set_default_emotion(EmotionState e) {

        // lock access

        const std::lock_guard<std::mutex> lock(emotion_config_default_mutex);

        // update our config, ignore duration as this is the default emotion

        emotion_config_default.select_config(e.value);

        emotion_target = &emotion_config_default;

        gaze_state_animation_restart = true;

        printf("> stored default emotion\n");

}

void Arbiter::set_current_emotion(EmotionState e) {

        // lock access

        const std::lock_guard<std::mutex> lock(emotion_config_current_mutex);

        // this will set a emotion override for a given time, after the timeout

        // we will return to the default emotion state:

        emotion_config_current.select_config(e.value);

        emotion_config_current.set_duration(e.duration);

        emotion_target = &emotion_config_current;

        gaze_state_animation_restart = true;

        printf("> stored current emotion (val=%d, duration = %dms)\n", e.value, e.duration);

}

void Arbiter::set_mouth_config(MouthConfig m) {

        // lock access

        const std::lock_guard<std::mutex> lock(mouth_config_override_mutex);

        mouth_config = m;

}

void Arbiter::set_gaze_target(humotion::GazeState target) {

        // lock access

        const std::lock_guard<std::mutex> lock(requested_gaze_state_mutex);

        requested_gaze_state = target;

}

void Arbiter::set_mouth_target(humotion::MouthState target) {

        // lock access

        const std::lock_guard<std::mutex> lock(requested_mouth_state_mutex);

        requested_mouth_state = target;

}

//! note: this is blocking!

void Arbiter::play_animation(boost::shared_ptr<Animation> incoming_animation) {

        // incoming animation, check if this would conflict with any pending animations:

        // lock access & iterate over vector:

        std::unique_lock<std::mutex> lock_av(active_animation_vector_mutex);

        active_animation_vector_t::iterator it;

        for (it = active_animation_vector.begin(); it < active_animation_vector.end(); it++) {

                boost::shared_ptr<Animation> current_ani = *it;

                // check if the running animation collides with the incoming animation:

                if (current_ani->collides_with(incoming_animation.get())) {

                        // this would fail, we can not play this animation right now!

                        throw runtime_error("incoming animation collides with unfinished active animation");

                        return;

                }

        }

        // ok, fine. no other animations are active that would collide with the new request

        // thus it is safe to enqueue this item:

        active_animation_vector.push_back(incoming_animation);

        // unlock mutex

        lock_av.unlock();

        // start playback!

        incoming_animation->start();

        // block until ani was played back

        while (incoming_animation->is_active()) {

                std::this_thread::sleep_for(std::chrono::milliseconds(1)); // save cpu cycles

        }

        // ok, it finished. we can safely remove it now:

        lock_av.lock();

        for (it = active_animation_vector.begin(); it < active_animation_vector.end();) {

                boost::shared_ptr<Animation> current_ani = *it;

                if (*it == incoming_animation) {

                        // printf(">match -> remove incoming ani again\n");

                        it = active_animation_vector.erase(it);

                }

                else {

                        ++it;

                }

        }

}

void Arbiter::speak(

   boost::shared_ptr<Utterance> u) { //, ao_sample_format audio_format, char *audio_data, unsigned int audio_len){

        // lock audio playback as such:

        const std::lock_guard<std::mutex> lock_audio(audio_player_mutex);

        {

                // lock utterance & store data:

                const std::lock_guard<std::mutex> lock(utterance_mutex);

                utterance = u;

        }

        // start audio playback, this function returns once we started playback

        if (audio_player == NULL) {

                printf("> audio_player disabled, not speaking '%s'\n", u.get()->get_text().c_str());

        }

        else {

                audio_player->play(utterance->get_audio_data());

        }

        utterance->start_playback();

        // wait until audio playback was finished:

        if (audio_player != NULL) {

                while (audio_player->is_playing()) {

                        // save some cpu cycles:

                        std::this_thread::sleep_for(std::chrono::milliseconds(1)); // 1ms

                }

        }

        // in case the audio output fails, we end up here before the utterance is finished.

        // so check now if the utterance finished as well:

        while (utterance->is_playing()) {

                // save some cpu cycles:

                std::this_thread::sleep_for(std::chrono::milliseconds(1));

        }

}

bool Arbiter::speak_active() {

        if (audio_player == NULL) {

                return false;

        }

        else {

                return audio_player->is_playing();

        }

}

void Arbiter::arbitrate() {

        // handle arbitration, DO NOT RE-ORDER these calls!

        // try to process all incoming requests

        override_by_emotion();

        // overwrite with mouth targets from outside:

        override_by_mouth();

        // lip animation

        override_by_utterance();

        // animations

        override_by_animation();

        // gaze target is set by requested_gaze_target

        override_by_gaze();

}

void Arbiter::override_by_utterance() {

        // fetch MouthState by utterance:

        if (!utterance->is_playing()) {

                // not playing -> return

                return;

        }

        // fetch symbol

        string symbol = utterance->currently_active_phoneme();

        if ((symbol.empty()) || (symbol == "")) {

                return;

        }

        // fetch config

        mouth_config.init_by_symbol(symbol);

        // overwrite mouth with offsets from utterance player

        mouth_config.apply_on_mouth_state(&mouth_state);

}

void Arbiter::override_by_mouth() {

        // external mouth requests can overwrite the mouth targets!

        if ((requested_mouth_state.position_center + requested_mouth_state.position_left + requested_mouth_state.position_right) >

            0.0) {

                // values given -> override emotion data

                mouth_state = requested_mouth_state;

        }

        // requested_mouth_state.dump();

        // mouth_state.dump();

}

void Arbiter::override_by_gaze() {

        // copy from requested target:

        gaze_state.pan = requested_gaze_state.pan;

        gaze_state.tilt = requested_gaze_state.tilt;

        gaze_state.roll = requested_gaze_state.roll;

        gaze_state.gaze_type = requested_gaze_state.gaze_type;

        gaze_state.timestamp = requested_gaze_state.timestamp;

        gaze_state.vergence = requested_gaze_state.vergence;

        // add offset values from user:

        gaze_state.pan_offset += requested_gaze_state.pan_offset;

        gaze_state.tilt_offset += requested_gaze_state.tilt_offset;

}

void Arbiter::override_by_emotion() {

        if (emotion_target == NULL) {

                return;

        }

        // lock access

        const std::lock_guard<std::mutex> lock1(emotion_config_default_mutex);

        // did the current emotion time out?

        if (!emotion_target->is_active()) {

                // revert to default:

                emotion_target = &emotion_config_default;

                // trigger soft fade

                gaze_state_animation_restart = true;

        }

        // emotions will set the mouth state here (will be overwritten by speak)

        mouth_state = emotion_target->mouth_override;

        // mouth_state = ...

        // emotions will add an offset to the gaze state:

        // pan,tilt,roll will be overwritten by override_by_gaze() we only keep the offset values from this!

        // FIXME: change to eyelid opening  upper/lower once humotion supports it

        // eyebrow angles come from this

        humotion::GazeState gaze_state_target = emotion_target->gaze_override;

        // apply a soft overblending to the new gaze target

        gaze_state = soft_overblend_gaze(gaze_state, gaze_state_target, emotion_target->overblend_time_ms);

        // gaze_state.dump();

        // IMPORTANT: clear request from emotion target:

        emotion_target->gaze_override.eyeblink_request_left = 0;

        emotion_target->gaze_override.eyeblink_request_right = 0;

}

humotion::GazeState Arbiter::soft_overblend_gaze(humotion::GazeState gaze_now, humotion::GazeState gaze_target,

                                                 unsigned int overblend_time) {

        humotion::GazeState result = gaze_now;

        result.eyeblink_request_left = 0;

        result.eyeblink_request_right = 0;

        if (gaze_state_animation_restart) {

                // new incoming target, set up soft fade:

                cout << "NEW EMOTION\n";

                gaze_state_old = gaze_state;

                gaze_state_end_time = std::chrono::steady_clock::now() + std::chrono::milliseconds(overblend_time);

                gaze_state_animation_restart = false;

        }

        // do smooth overblend, all targets should be reached at gaze_state_end_time

        double diff_ms =

           std::chrono::duration<double, std::milli>(gaze_state_end_time - std::chrono::steady_clock::now()).count();

        if (diff_ms < 0) {

                // animation is done, exit now

                return gaze_target;

        }

        else {

                // do smooth animation

                double tp = 1.0 - diff_ms / static_cast<double>(overblend_time);

                result.pan_offset = gaze_state_old.pan_offset + tp * (gaze_target.pan_offset - gaze_state_old.pan_offset);

                result.tilt_offset = gaze_state_old.tilt_offset + tp * (gaze_target.tilt_offset - gaze_state_old.tilt_offset);

                result.roll_offset = gaze_state_old.roll_offset + tp * (gaze_target.roll_offset - gaze_state_old.roll_offset);

                result.eyelid_opening_lower =

                   gaze_state_old.eyelid_opening_lower + tp * (gaze_target.eyelid_opening_lower - gaze_state_old.eyelid_opening_lower);

                result.eyelid_opening_upper =

                   gaze_state_old.eyelid_opening_upper + tp * (gaze_target.eyelid_opening_upper - gaze_state_old.eyelid_opening_upper);

                result.eyebrow_left = gaze_target.eyebrow_left;

                result.eyebrow_right = gaze_target.eyebrow_right;

        }

        return result;

}

void Arbiter::override_by_animation() {

        // lock access & iterate over vector:

        const std::lock_guard<std::mutex> lock_av(active_animation_vector_mutex);

        active_animation_vector_t::iterator it;

        for (it = active_animation_vector.begin(); it < active_animation_vector.end(); it++) {

                boost::shared_ptr<Animation> current_ani = *it;

                // gaze_state.dump();

                current_ani->apply_on_gazestate(&gaze_state);

                // gaze_state.dump();

        }

}

humotion::GazeState Arbiter::get_gaze_state() {

        // gaze_state.dump();

        return gaze_state;

}

humotion::MouthState Arbiter::get_mouth_state() {

        return mouth_state;

}