made congestion/offline behaviour more stable: do no declare online right after going...

[pingcheck] / src / host / hoststatus.cpp

/*
The software in this package is distributed under the GNU General
Public License version 2 (with a special exception described below).

A copy of GNU General Public License (GPL) is included in this distribution,
in the file COPYING.GPL.

As a special exception, if other files instantiate templates or use macros
or inline functions from this file, or you compile this file and link it
with other works to produce a work based on this file, this file
does not by itself cause the resulting work to be covered
by the GNU General Public License.

However the source code for this file must still be made available
in accordance with section (3) of the GNU General Public License.

This exception does not invalidate any other reasons why a work based
on this file might be covered by the GNU General Public License.
*/
#include "host/hoststatus.h"

#include <iostream>
#include <logfunc.hpp>

#include "boost_assert_handler.h"

using namespace std;
using I2n::Logger::GlobalLogger;

//-----------------------------------------------------------------------------
// HostStatus
//-----------------------------------------------------------------------------

/**
 * @param host_address The address of the host it has to analyze.
 * @param ping_fail_percentage_limit The percentage threshold of pings that can
 * fail.
 * @param ping_congestion_limit_percentage The percentage threshold of pings
 * that can fail due to line congestion
 * @param ping_duration_congestion_thresh Threshold in micro seconds that marks
 * the difference between a "normal" and a congested line
 * @param n_parallel_pings Number of pings that is sent for each IP
 * @param link_analyzer The object used to notify the status of the host.
 */
HostStatus::HostStatus(
        const string &host_address,
        const int ping_fail_limit_percentage,
        const int ping_congestion_limit_percentage,
        const int ping_duration_congestion_thresh,
        const int n_parallel_pings,
        const LinkStatusItem link_analyzer
) :
    HostAddress( host_address ),
    LinkAnalyzer( link_analyzer ),
    PingFailLimitPercentage( ping_fail_limit_percentage ),
    PingCongestionLimitPercentage( ping_congestion_limit_percentage ),
    PingDurationCongestionsThresh( ping_duration_congestion_thresh*1000000 ),
    ResolvedIpCount( 0 ),
    PingsPerformedCount( 0 ),
    PingsFailedCount( 0 ),
    PingCongestionCount( 0 ),
    ExceededPingFailedLimit( false ),
    ExceededPingCongestionLimit( false ),
    NParallelPingers( n_parallel_pings),
    InBurstMode( false )
{
    BOOST_ASSERT( !HostAddress.empty() );
    BOOST_ASSERT( ( 0 <= PingFailLimitPercentage )
                    && ( PingFailLimitPercentage <= 100 ) );
    BOOST_ASSERT( ( 0 <= PingCongestionLimitPercentage )
                    && ( PingCongestionLimitPercentage <= 100 ) );
}

HostStatus::~HostStatus()
{
}


void HostStatus::set_n_parallel_pings(const int n_parallel_pings)
{
    if (ExceededPingCongestionLimit)
        InBurstMode = true;
    else
        InBurstMode = true;

    if (NParallelPingers != n_parallel_pings)
    {
        NParallelPingers = n_parallel_pings;
        reset_ping_counters();
    }
    GlobalLogger.debug() << log_prefix() << "#pingers set";
}


std::string HostStatus::log_prefix()
{
    std::stringstream temp;
    temp << "Stat(" << HostAddress << "): "
        << PingsFailedCount << " fail," << PingCongestionCount << " cong/"
        << PingsPerformedCount << " pings/" << NParallelPingers << "*"
        << ResolvedIpCount << " IPs: ";
    return temp.str();
}

/**
 * @param resolved_ip_count The number of IPs resolved for the host.
 */
void HostStatus::set_resolved_ip_count( const int resolved_ip_count )
{
    BOOST_ASSERT( 0 <= resolved_ip_count );

    if (resolved_ip_count != ResolvedIpCount)
    {   // assume that the target has changed --> reset counters
        reset_ping_counters();
    }
    ResolvedIpCount = resolved_ip_count;

    GlobalLogger.debug() << log_prefix() << "#IPs set";
}

/**
 * @return true if the amount of failed pings given to the host exceeded the
 * limit.
 */
bool HostStatus::exceeded_ping_failed_limit() const
{
    return ExceededPingFailedLimit;
}

/**
 * @return true if the amount of congested pings given to the host exceeded the
 * limit.
 */
bool HostStatus::exceeded_ping_congestion_limit() const
{
    return ExceededPingCongestionLimit;
}

/**
 * Tells the status analyzer how the last ping went
 *
 * @param result: status of ping specifying success/failure and reason of fail
 * @param ping_duration_us duration of ping in micro seconds
 */
void HostStatus::update_ping_statistics( const PingStatus &result,
                                         const long ping_duration_us )
{
    float ping_duration_ms = static_cast<float>(ping_duration_us) / 1000.0f;

    GlobalLogger.debug() << log_prefix() << "add ping with result "
        << to_string(result) << " which took " << ping_duration_ms << " ms";

    BOOST_ASSERT( 0 <= ResolvedIpCount );
    BOOST_ASSERT( 0 <= PingsPerformedCount );
    BOOST_ASSERT( PingsFailedCount <= PingsPerformedCount );
    BOOST_ASSERT( PingCongestionCount <= PingsPerformedCount );

    increase_ping_performed_count();

    bool failed_because_congested = update_congestion_stats( result,
                                                             ping_duration_us );
    update_fail_stats( result, failed_because_congested );

    // after we tried all IPs resolved for this host, we can analyze how many
    // failed
    if ( tried_all_resolved_ip() )
    {
        analyze_ping_statistics();

        reset_ping_counters();
    }

    BOOST_ASSERT( PingsFailedCount <= PingsPerformedCount );
    BOOST_ASSERT( PingCongestionCount <= PingsPerformedCount );
}


void HostStatus::update_fail_stats( const PingStatus &result,
                                    const bool failed_because_congested )
{
    if ( result != PingStatus_SuccessReply
      && result != PingStatus_SuccessOutdatedIP
      && !failed_because_congested )
    {
        increase_ping_failed_count();
    }

    analyze_ping_failed_count();
}


bool HostStatus::update_congestion_stats( const PingStatus &result,
                                          const long ping_duration_us )
{
    bool is_congested = false;
    if (ping_duration_us > PingDurationCongestionsThresh)
        is_congested = true;
    else if ( result == PingStatus_FailureTimeout )
        is_congested = true;
    // PingStatus_FailureNoIP, PingStatus_SuccessOutdatedIP could also be caused
    // by congestion, but also by other reasons (e.g. firewall blocking port 53)

    if (is_congested)
        increase_ping_congestion_count();

    analyze_ping_congestion_count();

    return is_congested;
}


bool HostStatus::tried_all_resolved_ip() const
{
    BOOST_ASSERT( 0 < PingsPerformedCount );

    return ( PingsPerformedCount >= ResolvedIpCount*NParallelPingers );
}


/** @brief called when tried_all_resolved_ip() */
void HostStatus::analyze_ping_statistics()
{
    BOOST_ASSERT( !HostAddress.empty() );
    BOOST_ASSERT( PingsPerformedCount >= ResolvedIpCount*NParallelPingers );

    // timeouts are not counted towards failures, only count as congestions
    // However, if all pings timed out even in burst mode, then we still declare
    // the line down
    if (InBurstMode && PingCongestionCount >= PingsPerformedCount)
    {
        GlobalLogger.notice() << log_prefix() << "All pings timed out despite "
            << "using more pings per IP --> assume connection is really down";
        PingsFailedCount += PingCongestionCount;
        PingCongestionCount = 0;
        ExceededPingFailedLimit = true;
        ExceededPingCongestionLimit = false;
    }

    // notify if the amount of pings that failed exceed the limit
    if ( exceeded_ping_failed_limit() )
    {
        GlobalLogger.debug() << log_prefix() << "notify down";
        LinkAnalyzer->notify_host_down( HostAddress );
    }
    else if (exceeded_ping_congestion_limit() && !InBurstMode)
        // only notify up if will not try burst mode next
        // otherwise will continuously notify up and down if get timeouts
        GlobalLogger.notice() << log_prefix() << "will not notify up because "
            << " will go into burst mode next";
    else
    {
        GlobalLogger.debug() << log_prefix() << "notify up";
        LinkAnalyzer->notify_host_up( HostAddress );
    }

    // nothing else to do about congestion here, congestion is not forwarded to
    // central LinkAnalyzer
} //lint !e1762

void HostStatus::reset_ping_counters()
{
    PingsPerformedCount = 0;
    PingsFailedCount = 0;
    PingCongestionCount = 0;
}

void HostStatus::increase_ping_performed_count()
{
    ++PingsPerformedCount;

    BOOST_ASSERT( 0 < PingsPerformedCount );
}

void HostStatus::increase_ping_failed_count()
{
    ++PingsFailedCount;

    BOOST_ASSERT( ( 0 <= PingsFailedCount ) && ( PingsFailedCount <= PingsPerformedCount ) );
}

void HostStatus::increase_ping_congestion_count()
{
    ++PingCongestionCount;

    BOOST_ASSERT( ( 0 <= PingCongestionCount )
                    && ( PingCongestionCount <= PingsPerformedCount ) );
}

void HostStatus::analyze_ping_failed_count()
{
    BOOST_ASSERT( ( 0 <= PingFailLimitPercentage ) && ( PingFailLimitPercentage <= 100 ) );
    BOOST_ASSERT( ( 0 <= PingsFailedCount ) && ( PingsFailedCount <= PingsPerformedCount ) );

    int limit = ( PingsPerformedCount * PingFailLimitPercentage) / 100;

    // keep a boolean variable because the PingsFailedCount can be reseted
    if ( PingsFailedCount > limit )
    {
        ExceededPingFailedLimit = true;

        GlobalLogger.debug() << log_prefix() << "exceed fail limit=" << limit;
    }
    else
    {
        ExceededPingFailedLimit = false;

        GlobalLogger.debug() << log_prefix() << "below fail limit=" << limit;
    }
}

void HostStatus::analyze_ping_congestion_count()
{
    BOOST_ASSERT( ( 0 <= PingCongestionLimitPercentage )
                    && ( PingCongestionLimitPercentage <= 100 ) );
    BOOST_ASSERT( ( 0 <= PingCongestionCount )
                    && ( PingCongestionCount <= PingsPerformedCount ) );

    int limit = ( PingsPerformedCount * PingCongestionLimitPercentage) / 100;

    // keep a boolean variable because the PingCongestionCount can be reseted
    if ( PingCongestionCount > limit )
    {
        ExceededPingCongestionLimit = true;

        GlobalLogger.debug() << log_prefix() << "exceed congestion limit="
                             << limit;
    }
    else
    {
        ExceededPingCongestionLimit = false;

        GlobalLogger.debug() << log_prefix() << "below congestion limit="
                             << limit;
    }
}