developer.intra2net.com Git - pingcheck/blob - src/host/pingscheduler.cpp

   1 /*
   2 The software in this package is distributed under the GNU General
   3 Public License version 2 (with a special exception described below).
   4
   5 A copy of GNU General Public License (GPL) is included in this distribution,
   6 in the file COPYING.GPL.
   7
   8 As a special exception, if other files instantiate templates or use macros
   9 or inline functions from this file, or you compile this file and link it
  10 with other works to produce a work based on this file, this file
  11 does not by itself cause the resulting work to be covered
  12 by the GNU General Public License.
  13
  14 However the source code for this file must still be made available
  15 in accordance with section (3) of the GNU General Public License.
  16
  17 This exception does not invalidate any other reasons why a work based
  18 on this file might be covered by the GNU General Public License.
  19 */
  20 #include "host/pingscheduler.h"
  21
  22 #include <iostream>
  23 #include <limits>
  24
  25 #include <boost/bind.hpp>
  26 #include <boost/foreach.hpp>
  27
  28 #include <logfunc.hpp>
  29
  30 #include "boost_assert_handler.h"
  31 #include "host/pingerfactory.h"
  32 #include "dns/dnsmaster.h"
  33 #include "icmp/icmppinger.h"
  34 #include "link/linkstatus.h"
  35
  36 using namespace std;
  37 using boost::asio::io_service;
  38 using boost::bind;
  39 using boost::date_time::time_resolution_traits_adapted64_impl;
  40 using boost::posix_time::microsec_clock;
  41 using boost::posix_time::ptime;
  42 using boost::posix_time::seconds;
  43 using boost::posix_time::milliseconds;
  44 using boost::shared_ptr;
  45 using I2n::Logger::GlobalLogger;
  46
  47
  48 //-----------------------------------------------------------------------------
  49 // PingScheduler
  50 //-----------------------------------------------------------------------------
  51
  52 /**
  53  * @brief Parameterized constructor.
  54  *
  55  * @param io_serv The one @c io_serv object that controls async processing
  56  * @param network_interface The name of the network interface sending the pings.
  57  * @param destination_address The remote address to ping.
  58  * @param destination_port The remote port to ping.
  59  * @param ping_protocol_list A list of protocols to use.
  60  * @param ping_interval_in_sec Amount of time between each ping.
  61  * @param ping_fail_percentage_limit Maximum amount of pings that can fail.
  62  * @param ping_congestion_percentage_limit Amount of pings indication congested
  63  * line
  64  * @param ping_congestion_duration_thresh Duration in seconds that indicates a
  65  * congested line
  66  * @param ping_reply_timeout Max amount time to wait for ping to finish
  67  * @param link_analyzer The object to monitor the link status.
  68  * @param first_delay Delay in seconds from start_pinging to first ping attempt
  69  * @param n_parallel_pings: Number of pingers to ping the same IP in parallel
  70  */
  71 PingScheduler::PingScheduler(
  72         const IoServiceItem io_serv,
  73         const string &network_interface,
  74         const string &destination_address,
  75         const uint16_t destination_port,
  76         const PingProtocolList &ping_protocol_list,
  77         const long ping_interval_in_sec,
  78         const int ping_fail_percentage_limit,
  79         const int ping_congestion_percentage_limit,
  80         const int congest_caused_by_fail_percentage_limit,
  81         const int ping_congestion_duration_thresh,
  82         const int ping_reply_timeout,
  83         LinkStatusItem link_analyzer,
  84         const int first_delay,
  85         const int n_parallel_pings,
  86         const int parallel_ping_delay,
  87         const int ping_timeout_factor
  88 ) :
  89     IoService( io_serv ),
  90     NetworkInterfaceName( network_interface ),
  91     DestinationAddress( destination_address ),
  92     DestinationPort( destination_port ),
  93     Protocols( ping_protocol_list ),
  94     ProtocolIter(),
  95     PingIntervalInSec( ping_interval_in_sec ),
  96     NPingers( n_parallel_pings ),
  97     FirstDelay( first_delay ),
  98     NextPingTimer( *io_serv ),
  99     TimeSentLastPing( microsec_clock::universal_time() ),
 100     PingReplyTimeout( ping_reply_timeout ),
 101     PingReplyTimeoutOrig( ping_reply_timeout ),
 102     HostAnalyzer( destination_address, ping_fail_percentage_limit,
 103                   ping_congestion_percentage_limit,
 104                   congest_caused_by_fail_percentage_limit,
 105                   ping_congestion_duration_thresh, n_parallel_pings,
 106                   link_analyzer ),
 107     Resolver(),
 108     Pingers(),
 109     NPingersDone( 0 ),
 110     ParallelPingDelay( parallel_ping_delay ),
 111     DelayedPingTimer( *io_serv ),
 112     WantToPing( false ),
 113     LogPrefix(),
 114     ContinueOnOutdatedIps( false ),
 115     PingTimeoutFactor( ping_timeout_factor )
 116 {
 117     BOOST_ASSERT( !network_interface.empty() );
 118     BOOST_ASSERT( !destination_address.empty() );
 119     BOOST_ASSERT( ( 0 < destination_port ) &&
 120                   ( destination_port < numeric_limits<uint16_t>::max() ) );
 121     BOOST_ASSERT( 0 < ping_interval_in_sec );
 122     BOOST_ASSERT( (0 <= ping_fail_percentage_limit) &&
 123                   ( ping_fail_percentage_limit <= 100) );
 124
 125     update_log_prefix();
 126
 127     init_ping_protocol();
 128
 129     // start resolving already so we are prepared to ping
 130     update_dns_resolver();
 131
 132 }
 133
 134 /**
 135  * @brief Destructor.
 136  */
 137 PingScheduler::~PingScheduler()
 138 {
 139 }
 140
 141 void PingScheduler::stop_pinging()
 142 {
 143     // stop pinger and resolver
 144     GlobalLogger.debug() << LogPrefix << "scheduler: stop pinging";
 145     clear_pingers();
 146     cancel_resolve(true);
 147
 148     // now cancel the own timer in case that pinger cancelation called callback
 149     GlobalLogger.debug() << LogPrefix << "scheduler: cancel timer";
 150     NextPingTimer.cancel();
 151 }
 152
 153 /**
 154  * @brief stop all pingers and remove them from Pingers variable which will
 155  *   proboably cause their destruction
 156  *
 157  * Pingers is empty afterwards
 158  */
 159 void PingScheduler::clear_pingers()
 160 {
 161     PingerItem pinger;
 162     while(!Pingers.empty())
 163     {
 164         pinger = Pingers.back();
 165         pinger->stop_pinging();
 166         Pingers.pop_back();
 167     }
 168 }
 169
 170 /**
 171  * @brief Start into infinite loop of calls to ping
 172  *
 173  * Does not start yet but set NextPingTimer (possibly to 0), so action starts
 174  *   when io_service is started
 175  */
 176 void PingScheduler::start_pinging()
 177 {
 178     if ( FirstDelay > 0 )
 179         GlobalLogger.info() << LogPrefix << "Delaying first ping by "
 180                                          << FirstDelay << "s";
 181     else
 182         GlobalLogger.info() << LogPrefix << "Schedule ping as soon as possible";
 183
 184     (void) NextPingTimer.expires_from_now( seconds( FirstDelay ) );
 185     NextPingTimer.async_wait( bind( &PingScheduler::ping, this,
 186                                           boost::asio::placeholders::error ) );
 187 }
 188
 189
 190 /**
 191  * @brief call Ping::ping and schedule a call to ping_done_handler when finished
 192  */
 193 void PingScheduler::ping(const boost::system::error_code &error)
 194 {
 195     if ( error )
 196     {   // get here, e.g. by NextPingTimer.cancel in stop_pinging
 197         if ( error ==  boost::asio::error::operation_aborted )
 198             GlobalLogger.error() << LogPrefix << "Timer for ping was cancelled!"
 199                                  << " --> Stopping";
 200         else
 201             GlobalLogger.error() << LogPrefix << "Received error " << error
 202                                  << " waiting for ping! Stopping";
 203         return;
 204     }
 205
 206     // ping as soon as dns is ready
 207     WantToPing = true;
 208     ping_when_ready();
 209 }
 210
 211
 212 void PingScheduler::ping_when_ready()
 213 {
 214     if ( !WantToPing )
 215     {
 216         GlobalLogger.info() << LogPrefix << "waiting for ping request "
 217             << "(should take no more than " << PingIntervalInSec << "s)";
 218         return;
 219     }
 220     else if ( Resolver && Resolver->is_resolving() )
 221     {
 222         GlobalLogger.info() << LogPrefix << "waiting for DNS to finish";
 223         return;
 224     }
 225     else if ( !Resolver )
 226     {
 227         // should not happen, but check anyway
 228         GlobalLogger.warning() << LogPrefix << "Have no resolver!";
 229         return;
 230     }
 231
 232     GlobalLogger.info() << LogPrefix << "start ping";
 233     WantToPing = false;
 234
 235     // try to get an up-to-date IP (ContinueOnOutdatedIps may only be set
 236     //   because a CNAME was out of date -- IPs may still be current)
 237     HostAddress ip = Resolver->get_next_ip();
 238
 239     if ( !ip.is_valid() )
 240     {   // this can happen in 2 cases: if ContinueOnOutdatedIps==true
 241         // or when ip went out of date between resolve and now
 242         // --> try to use outdated IP
 243         GlobalLogger.info() << LogPrefix << "Checking for outdated IPs";
 244         bool check_up_to_date = false;
 245         ip = Resolver->get_next_ip(check_up_to_date);
 246     }
 247     if ( !ip.is_valid() )
 248     {   // Do not even have an outdated IP!
 249         // This happens if have no cached IPs and resolve failed
 250         GlobalLogger.info() << LogPrefix << "Not even outdated IP to ping "
 251             << "-- treat like a failed ping.";
 252
 253         // skip the ping and directly call ping_done_handler the appropriate
 254         // number of times
 255         for (int count=0; count<NPingers; ++count)
 256             ping_done_handler(PingStatus_FailureNoIP, 0);
 257     }
 258     else
 259     {
 260         // create new pingers
 261         for (int count=0; count<NPingers; ++count)
 262             Pingers.push_back(
 263                     PingerFactory::createPinger(*ProtocolIter, IoService,
 264                                                 NetworkInterfaceName,
 265                                                 PingReplyTimeout) );
 266
 267         // remember when pinging started
 268         TimeSentLastPing = microsec_clock::universal_time();
 269
 270         // get actual IP
 271         boost::asio::ip::address actual_ip = ip.get_ip();
 272         GlobalLogger.info() << LogPrefix << "pinging IP " << actual_ip
 273             << " with TTL " << ip.get_ttl().get_updated_value() << "s";
 274         delayed_ping(boost::system::error_code(), actual_ip, 0);
 275     }
 276 }
 277
 278 void PingScheduler::delayed_ping( const boost::system::error_code &error,
 279                                   const boost::asio::ip::address ip,
 280                                   const int pinger_index )
 281 {
 282     if (error)
 283     {
 284         GlobalLogger.info() << LogPrefix << "delayed ping received an error: "
 285                                          << error;
 286         return;
 287     }
 288
 289     GlobalLogger.debug() << LogPrefix << "starting delayed ping index "
 290                           << pinger_index;
 291     Pingers[pinger_index]->ping(ip,
 292                                 DestinationPort,
 293                                 boost::bind(&PingScheduler::ping_done_handler,
 294                                                                 this, _1, _2) );
 295     if (pinger_index >= NPingers-1)
 296         GlobalLogger.debug() << LogPrefix << "started all delayed pings";
 297     else
 298     {
 299         DelayedPingTimer.expires_from_now( milliseconds(ParallelPingDelay) );
 300         DelayedPingTimer.async_wait( bind( &PingScheduler::delayed_ping,
 301                   this, boost::asio::placeholders::error, ip, pinger_index+1) );
 302     }
 303 }
 304
 305
 306 //------------------------------------------------------------------------------
 307 // Post Processing of Ping result and Preparation for next ping
 308 //------------------------------------------------------------------------------
 309
 310 /**
 311  * @brief called when Ping::ping is done; calls functions to update
 312  *   statistics, ping interval and elapsed time;
 313  *   schedules a call to ping, thereby closing the loop
 314  */
 315 void PingScheduler::ping_done_handler( const PingStatus &result,
 316                                        const long ping_duration_us )
 317 {
 318     PingStatus edited_result = result;
 319     if (result == PingStatus_SuccessReply && ContinueOnOutdatedIps)
 320     {
 321         edited_result = PingStatus_SuccessOutdatedIP;
 322
 323         // reset ContinueOnOutdatedIps
 324         ContinueOnOutdatedIps = false;
 325         update_log_prefix();
 326     }
 327
 328     ++NPingersDone;
 329     GlobalLogger.info() << LogPrefix << "Ping " << NPingersDone << " of "
 330         << NPingers << " done with result " << to_string(edited_result);
 331
 332     // post-processing
 333     // can call update_ping_interval only after update_ping_statistics!
 334     HostAnalyzer.update_ping_statistics( edited_result, ping_duration_us );
 335
 336     // prepare next ping only after all pingers are done
 337     if (NPingersDone == NPingers)
 338     {
 339         // stop and destruct all pingers
 340         clear_pingers();
 341
 342         GlobalLogger.debug() << LogPrefix
 343             << "--------------------------------------------------------------";
 344
 345         // update variables for next ping: number of pings, delay, protocol
 346         update_ping_protocol();
 347         update_ping_interval();
 348         update_ping_number();
 349
 350         prepare_next_ping();
 351     }
 352 }
 353
 354
 355 void PingScheduler::prepare_next_ping()
 356 {
 357     // start DNS resolve if necessary
 358     update_dns_resolver();
 359
 360     NPingersDone = 0;
 361
 362     // schedule next ping
 363     int seconds_since_last_ping = (microsec_clock::universal_time()
 364                                             - TimeSentLastPing).total_seconds();
 365     if ( seconds_since_last_ping > PingIntervalInSec )
 366     {
 367         GlobalLogger.info() << "We are late for next ping!";
 368         seconds_since_last_ping = PingIntervalInSec;
 369         (void) NextPingTimer.expires_from_now( seconds(0) );
 370     }
 371     else
 372         (void) NextPingTimer.expires_from_now( seconds( PingIntervalInSec
 373                                                   - seconds_since_last_ping ) );
 374     NextPingTimer.async_wait( bind( &PingScheduler::ping, this,
 375                                          boost::asio::placeholders::error ) );
 376 }
 377
 378 void PingScheduler::update_ping_interval()
 379 {
 380     // have to ping more often?
 381     if ( HostAnalyzer.exceeded_ping_failed_limit() )
 382     {
 383         PingIntervalInSec.speed_up();
 384
 385         GlobalLogger.debug() << LogPrefix << "- Speeding up ping interval to: "
 386                              << PingIntervalInSec << "s";
 387         PingReplyTimeout = PingReplyTimeoutOrig * PingTimeoutFactor;
 388         GlobalLogger.debug() << LogPrefix << "- Increase ping timeout to "
 389                              << PingReplyTimeout << "s";
 390     }
 391     else
 392     {
 393         PingIntervalInSec.back_to_original();
 394
 395         GlobalLogger.debug() << LogPrefix << "- Stick to the original ping "
 396                              << "interval: " << PingIntervalInSec << "s";
 397         PingReplyTimeout = PingReplyTimeoutOrig;
 398         GlobalLogger.debug() << LogPrefix << "- Reset ping timeout to "
 399                              << PingReplyTimeout << "s";
 400     }
 401 }
 402
 403 /** in case of congested line, increase number of pings
 404  *
 405  * CAUTION! Only call this after clear_pingers !!!
 406  * */
 407 void PingScheduler::update_ping_number()
 408 {
 409     // make sure we do not loose track of pingers here
 410     if ( NPingersDone != NPingers  || !Pingers.empty() )
 411     {
 412         GlobalLogger.warning() << LogPrefix << "Should only change number of "
 413             << "pingers when all are finished and deleted! Have " << NPingers
 414             << " pingers, " << NPingersDone << " of which are done and "
 415             << Pingers.size() << " in listDone! Will not change NPingers.";
 416         return;
 417     }
 418
 419     if ( HostAnalyzer.exceeded_ping_congestion_limit() )
 420     {
 421         NPingers.increase();
 422
 423         GlobalLogger.notice() << LogPrefix << "Line appears congested!";
 424         GlobalLogger.debug() << LogPrefix << "- Increasing ping number to: "
 425                              << NPingers;
 426     }
 427     else
 428     {
 429         NPingers.back_to_original();
 430
 431         GlobalLogger.debug() << LogPrefix << "- Stick to the original ping "
 432                              << "number: " << NPingers;
 433     }
 434
 435     HostAnalyzer.set_n_parallel_pings(NPingers);
 436 }
 437
 438 //------------------------------------------------------------------------------
 439 // Ping Protocol Rotation
 440 //------------------------------------------------------------------------------
 441
 442 void PingScheduler::init_ping_protocol()
 443 {
 444     ProtocolIter = Protocols.end();
 445     get_next_ping_protocol();
 446 }
 447
 448 void PingScheduler::update_ping_protocol()
 449 {
 450     if ( can_change_ping_protocol() )
 451     {
 452         get_next_ping_protocol();
 453     }
 454 }
 455
 456 void PingScheduler::get_next_ping_protocol()
 457 {
 458     ++ProtocolIter;
 459     if (ProtocolIter == Protocols.end())
 460         ProtocolIter = Protocols.begin();
 461 }
 462
 463 bool PingScheduler::can_change_ping_protocol() const
 464 {
 465     // TODO can_change_ping_protocol() and get_next_ping_protocol() may be
 466     // implemented in a Algorithm class that can be exchanged in this class to
 467     // provide an algorithm neutral class
 468     return true;
 469 }
 470
 471 //------------------------------------------------------------------------------
 472 // DNS host name resolution
 473 //------------------------------------------------------------------------------
 474
 475 // show "!" after host name if running on outdated IPs
 476 void PingScheduler::update_log_prefix()
 477 {
 478     std::stringstream temp;
 479     temp << "Sched(" << DestinationAddress;
 480     if (ContinueOnOutdatedIps)
 481         temp << "!";
 482     temp << "): ";
 483     LogPrefix = temp.str();
 484 }
 485
 486 void PingScheduler::update_dns_resolver()
 487 {
 488     if (Resolver && Resolver->is_resolving())
 489         cancel_resolve(false);
 490
 491     if (ContinueOnOutdatedIps)
 492     {
 493         ContinueOnOutdatedIps = false;
 494         update_log_prefix();
 495     }
 496
 497     // DNS master caches created resolvers and resolved IPs, so this will
 498     // probably just return an existing resolver with already resolved IPs for
 499     // requested protocol ( ICMP/TCP is ignored, only IPv4/v6 is important)
 500     Resolver = DnsMaster::get_instance()->get_resolver_for(DestinationAddress,
 501                                                            *ProtocolIter);
 502
 503     // get number of up-to-date IPs
 504     // TODO should check here, if they will be up to date in PingIntervalInSec
 505     bool check_up_to_date = true;
 506     int ip_count = Resolver->get_resolved_ip_count(check_up_to_date);
 507     if (ip_count > 0)
 508     {
 509         GlobalLogger.info() << LogPrefix << "Set resolved_ip_count to "
 510             << ip_count << " (IPs may be outdated=" << !check_up_to_date << ")";
 511         HostAnalyzer.set_resolved_ip_count( ip_count );
 512
 513         if (Resolver->is_resolving())
 514             GlobalLogger.warning() << LogPrefix << "have up to date IPs but "
 515                 << "resolver seems to be resolving all the same... "
 516                 << "Start pinging anyway!";
 517         ping_when_ready();
 518     }
 519     else
 520     {
 521         GlobalLogger.info() << LogPrefix
 522                             << "No up-to-date IPs --> start resolve";
 523         start_resolving_ping_address();
 524         // set resolved_ip_count will be called in resolve callback
 525     }
 526 }
 527
 528 void PingScheduler::start_resolving_ping_address()
 529 {
 530     Resolver->async_resolve( boost::bind(&PingScheduler::dns_resolve_callback,
 531                                           this, _1, _2) );
 532 }
 533
 534 void PingScheduler::dns_resolve_callback(const bool was_success,
 535                                          const int recursion_count)
 536 {
 537     GlobalLogger.info() << LogPrefix << "dns resolution finished "
 538                         << "with success = " << was_success << " "
 539                         << "after " << recursion_count << " recursions";
 540
 541     if ( was_success )
 542     {
 543         // trust that a successfull DNS resolve means we have an IP with TTL>0
 544         int ip_count = Resolver->get_resolved_ip_count(!ContinueOnOutdatedIps);
 545         if (ip_count == 0)
 546         {
 547             GlobalLogger.warning() << LogPrefix
 548                 << "Should not have reached this case: resolve was "
 549                 << "successfull but still have no IPs (up-to-date="
 550                 << !ContinueOnOutdatedIps << ")!";
 551             if (DnsMaster::get_instance()->get_resolved_ip_ttl_threshold() > 0)
 552                 GlobalLogger.warning() << LogPrefix << "This probably happened "
 553                     << "because you specified a TTL threshold > 0 but resolving"
 554                     << " had no effect on TTLs since external cache is only "
 555                     << "updated when TTL=0 is reached.";
 556         }
 557         else
 558         {
 559             GlobalLogger.info() << LogPrefix << "Set resolved_ip_count to "
 560                 << ip_count << " (IPs may be outdated="
 561                 << ContinueOnOutdatedIps << ") --> could ping now";
 562         }
 563         HostAnalyzer.set_resolved_ip_count( ip_count );
 564         ping_when_ready();
 565     }
 566     else
 567     {   // host name resolution failed; try again bypassing first outdated CNAME
 568         // or using cached IP
 569         std::string skip_host = Resolver->get_skip_cname();
 570
 571         if (skip_host.empty())
 572         {   // try to continue with cached IPs
 573             int ip_count = Resolver->get_resolved_ip_count(false);
 574
 575             if (ip_count == 0)
 576                 GlobalLogger.notice() << LogPrefix << "DNS failed "
 577                     << "and have no cached IPs either --> cannot ping";
 578                 // ping_when_ready will deal with this case
 579             else
 580             {
 581                 ContinueOnOutdatedIps = true;
 582                 update_log_prefix();
 583
 584                 GlobalLogger.notice() << LogPrefix << "DNS failed, "
 585                     << "try anyway with cached data";
 586             }
 587
 588             GlobalLogger.info() << LogPrefix << "Set resolved_ip_count to "
 589                 << ip_count << " (IPs may be outdated=" << true << ")";
 590             HostAnalyzer.set_resolved_ip_count( ip_count );
 591
 592             ping_when_ready();
 593         }
 594         else
 595         {   // have CNAME to continue
 596             ContinueOnOutdatedIps = true;
 597             update_log_prefix();
 598             GlobalLogger.notice() << LogPrefix << "DNS failed, "
 599                 << "try again skipping a CNAME and resolving "
 600                 << skip_host << " directly";
 601
 602             cancel_resolve(false);
 603
 604             // now create new resolver
 605             Resolver = DnsMaster::get_instance()
 606                                    ->get_resolver_for(skip_host, *ProtocolIter);
 607             start_resolving_ping_address();
 608         }
 609     }
 610 }
 611
 612 /**
 613  * cancel resolver if force_cancel or if it is not resolving DestinationAddress
 614  *
 615  * Resolvers have a life on their own: they are cached by DnsMaster so never go
 616  *   out of scope and even after calling callbacks, there might still be a
 617  *   longterm timer active to re-try resolving.
 618  * We want to cancel that long-term timer only if the Resolver is not for our
 619  *   real, original DestinationAddress but a CNAME, which can happen when trying
 620  *   to skip cnames and working on out-dated IPs
 621  */
 622 void PingScheduler::cancel_resolve(const bool force_cancel)
 623 {
 624     if (force_cancel)
 625     {
 626         GlobalLogger.info() << "Cancelling resolver (forced)";
 627         Resolver->cancel_resolve();
 628     }
 629     else if ( Resolver->get_hostname() == DestinationAddress )
 630         GlobalLogger.info() << LogPrefix
 631                             << "Leave original resolver active in background";
 632     else
 633     {
 634         GlobalLogger.info() << LogPrefix << "Cancel resolver for "
 635             << Resolver->get_hostname() << " since is not the original "
 636             << DestinationAddress;
 637         Resolver->cancel_resolve();
 638     }
 639 }
 640