bugfixed erase and re-set of TTLs; changed time warp thresh from 24h to 10mins

[pingcheck] / src / dns / dnscache.cpp

/*
 The software in this package is distributed under the GNU General
 Public License version 2 (with a special exception described below).

 A copy of GNU General Public License (GPL) is included in this distribution,
 in the file COPYING.GPL.

 As a special exception, if other files instantiate templates or use macros
 or inline functions from this file, or you compile this file and link it
 with other works to produce a work based on this file, this file
 does not by itself cause the resulting work to be covered
 by the GNU General Public License.

 However the source code for this file must still be made available
 in accordance with section (3) of the GNU General Public License.

 This exception does not invalidate any other reasons why a work based
 on this file might be covered by the GNU General Public License.

 Christian Herdtweck, Intra2net AG 2015
 */

#include "dns/dnscache.h"

#include <sstream>
#include <fstream>
#include <logfunc.hpp>
#include <filefunc.hxx>   // I2n::file_exists
#include <boost/foreach.hpp>
#include <boost/bind.hpp>
#include <boost/asio/placeholders.hpp>
#include <boost/serialization/serialization.hpp>
#include <boost/serialization/map.hpp>
#include <boost/serialization/string.hpp>
#include <boost/serialization/vector.hpp>
#include <boost/archive/xml_oarchive.hpp>
#include <boost/archive/xml_iarchive.hpp>
#include <boost/date_time/gregorian/gregorian_types.hpp>

#include "dns/dnsmaster.h"

using boost::bind;
using boost::posix_time::seconds;
using I2n::Logger::GlobalLogger;


namespace Config
{
    const int SAVE_TIMER_SECONDS = 60;
    const int CACHE_TIME_WARP_THRESH_MINS = 10;
    const int CACHE_REMOVE_OUTDATED_DAYS = 60;
}


// -----------------------------------------------------------------------------
// DNS Cache constructor / destructor
// -----------------------------------------------------------------------------

const string DnsCache::DoNotUseCacheFile = "do not use cache file!";

DnsCache::DnsCache(const IoServiceItem &io_serv,
                   const std::string &cache_file,
                   const uint32_t min_time_between_resolves)
    : IpCache()
    , CnameCache()
    , SaveTimer( *io_serv )
    , CacheFile( cache_file )
    , HasChanged( false )
    , MinTimeBetweenResolves( min_time_between_resolves )
{
    // load cache from file
    load_from_cachefile();

    // schedule next save
    (void) SaveTimer.expires_from_now( seconds( Config::SAVE_TIMER_SECONDS ) );
    SaveTimer.async_wait( bind( &DnsCache::schedule_save, this,
                                boost::asio::placeholders::error ) );
}


DnsCache::~DnsCache()
{
    GlobalLogger.info() << "DnsCache: being destructed";

    // save one last time without re-scheduling the next save
    save_to_cachefile();

    // cancel save timer
    SaveTimer.cancel();
}


// -----------------------------------------------------------------------------
// LOAD / SAVE
// -----------------------------------------------------------------------------

void DnsCache::schedule_save(const boost::system::error_code &error)
{
    if ( error ==  boost::asio::error::operation_aborted )   // cancelled
    {
        GlobalLogger.info() << "DnsCache: SaveTimer was cancelled "
                            << "--> no save and no re-schedule of saving!";
        return;
    }
    else if (error)
    {
        GlobalLogger.info() << "DnsCache: Received error " << error
                            << " in schedule_save "
                            << "--> no save now but re-schedule saving";
    }
    else
        save_to_cachefile();

    // schedule next save
    (void) SaveTimer.expires_from_now( seconds( Config::SAVE_TIMER_SECONDS ) );
    SaveTimer.async_wait( bind( &DnsCache::schedule_save, this,
                                boost::asio::placeholders::error ) );
}

void DnsCache::save_to_cachefile()
{
    if (!HasChanged)
        GlobalLogger.info() << "DnsCache: skip saving because has not changed";
    else if (CacheFile.empty())
        GlobalLogger.info()
                           << "DnsCache: skip saving because file name empty!";
    else if (CacheFile == DoNotUseCacheFile)
        GlobalLogger.info() << "DnsCache: configured not to use cache file";
    else
    {
        try
        {
            // clean up: remove very old entries
            remove_old_entries();

            // remember time of save
            std::string cache_save_time_str = boost::posix_time::to_iso_string(
                            boost::posix_time::second_clock::universal_time() );

            std::ofstream ofs( CacheFile.c_str() );
            boost::archive::xml_oarchive oa(ofs);
            oa & BOOST_SERIALIZATION_NVP(IpCache);
            oa & BOOST_SERIALIZATION_NVP(CnameCache);
            oa & BOOST_SERIALIZATION_NVP(cache_save_time_str);
            GlobalLogger.info() << "DnsCache: saved to cache file "
                                << CacheFile;

            HasChanged = false;
        }
        catch (std::exception &exc)
        {
            GlobalLogger.warning() << "DnsCache: Saving failed: " << exc.what();
        }
    }
}

void DnsCache::load_from_cachefile()
{
    if (CacheFile.empty())
        GlobalLogger.info()
                   << "DnsCache: cannot load because cache file name is empty!";
    else if (CacheFile == DoNotUseCacheFile)
        GlobalLogger.info() << "DnsCache: configured not to use cache file";
    else if ( !I2n::file_exists(CacheFile) )
        GlobalLogger.warning() << "DnsCache: cannot load because cache file "
                               << CacheFile << " does not exist!";
    else
    {
        try
        {
            std::ifstream ifs( CacheFile.c_str() );
            boost::archive::xml_iarchive ia(ifs);

            std::string cache_save_time_str;

            ia & BOOST_SERIALIZATION_NVP(IpCache);
            ia & BOOST_SERIALIZATION_NVP(CnameCache);
            ia & BOOST_SERIALIZATION_NVP(cache_save_time_str);

            boost::posix_time::ptime cache_save_time
                    = boost::posix_time::from_iso_string(cache_save_time_str);
            GlobalLogger.info() << "DnsCache: loaded from file " << CacheFile;

            check_timestamps(cache_save_time);
        }
        catch (boost::archive::archive_exception &exc)
        {
            GlobalLogger.warning()
                << "DnsCache: archive exception loading from " << CacheFile
                << ": " << exc.what();
        }
        catch (std::exception &exc)
        {
            GlobalLogger.warning() << "DnsCache: exception while loading from "
                                   << CacheFile << ": " << exc.what();
        }
    }
}


/**
 * @brief check that loaded cache really is from the past
 *
 * Added this to avoid trouble in case the system time is changed into the past.
 * In that cast would have TTLs here in cache that are valid much too long.
 *
 * Therefore check SaveCacheTime and if that is in the future, set all TTLs to 0
 *
 * @returns true if had to re-set timestamps
 */
bool DnsCache::check_timestamps(const boost::posix_time::ptime &cache_save_time)
{
    // check if CacheSaveTime is in the future
    boost::posix_time::ptime now
             = boost::posix_time::second_clock::universal_time()
             + boost::posix_time::minutes(Config::CACHE_TIME_WARP_THRESH_MINS);
    if (now > cache_save_time)
    {   // Cache was saved in the past -- everything alright
        return false;
    }

    GlobalLogger.warning() << "DnsCache: loaded cache from the future (saved "
                           << cache_save_time << ")! Resetting all TTLs to 0.";

    // reset TTLs in IP cache
    BOOST_FOREACH( ip_map_type::value_type &key_and_ip, IpCache )
    {
        BOOST_FOREACH( HostAddress &address, key_and_ip.second )
            address.get_ttl().set_value(0);
    }

    // reset TTLs in CNAME cache
    BOOST_FOREACH( cname_map_type::value_type &key_and_cname, CnameCache )
        key_and_cname.second.Ttl.set_value(0);

    HasChanged = true;

    //debug_print();

    return true;
}


/**
 * @brief remove entries from cache that are older than a certain threshold
 *
 * this also removes entres from IpCache with no IPs
 */
void DnsCache::remove_old_entries()
{
    boost::posix_time::ptime thresh
             = boost::posix_time::second_clock::universal_time()
             - boost::gregorian::days( Config::CACHE_REMOVE_OUTDATED_DAYS );

    // IP cache
    {
        ip_map_type::iterator it = IpCache.begin();
        ip_map_type::iterator it_end = IpCache.end();
        bool some_ip_up_to_date;
        while (it != it_end)
        {
            some_ip_up_to_date = false;
            BOOST_FOREACH( const HostAddress &address, (*it).second )
            {
                if ( ! address.get_ttl().was_set_before(thresh) )
                {
                    some_ip_up_to_date = true;
                    break;
                }
            }

            if ( ! some_ip_up_to_date )
            {
                GlobalLogger.debug() << "DnsCache: Removing empty/outdated IP "
                                     << "list for " << (*it).first.first;
                IpCache.erase( (*it++).first );
            }
            else
                ++it;
        }
    }

    // CNAME cache
    {
        cname_map_type::iterator it = CnameCache.begin();
        cname_map_type::iterator it_end = CnameCache.end();
        while (it != it_end)
        {
            if ( (*it).second.Ttl.was_set_before( thresh ) )
            {
                GlobalLogger.debug() << "DnsCache: Removing outdated CNAME for "
                                     << (*it).first;
                CnameCache.erase( (*it++).first );
            }
            else
                ++it;
        }
    }
}

// -----------------------------------------------------------------------------
// UPDATE
// -----------------------------------------------------------------------------

/*
 * warn if hostname is empty and remove trailing dot
 * also warn if protocol is neither IPv4 nor IPv6
 */
ip_map_key_type DnsCache::key_for_ips(const std::string &hostname,
                                   const DnsIpProtocol &protocol) const
{
    if (hostname.empty())
    {
        GlobalLogger.info() << "DnsCache: empty host!";
        return ip_map_key_type("", DNS_IPALL);
    }
    if (protocol == DNS_IPALL)
    {
        GlobalLogger.info() << "DnsCache: neither IPv4 nor v6!";
        return ip_map_key_type("", DNS_IPALL);
    }

    // check whether last character is a dot
    if (hostname.rfind('.') == hostname.length()-1)
        return ip_map_key_type( hostname.substr(0, hostname.length()-1),
                             protocol );
    else
        return ip_map_key_type( hostname,
                             protocol );
}


void DnsCache::update(const std::string &hostname,
                      const DnsIpProtocol &protocol,
                      const HostAddressVec &new_ips)
{
    // check for valid input arguments
    ip_map_key_type key = key_for_ips(hostname, protocol);
    if ( key.first.empty() )
        return;

    // ensure that there is never IP and CNAME for the same host
    if ( !get_cname(hostname).Host.empty() )
    {
        GlobalLogger.info() << "DnsCache: Saving IPs for " << key.first
            << " removes CNAME to " << get_cname(hostname).Host << "!";
        update(hostname, Cname());   // overwrite with "empty" cname
    }

    // ensure min ttl of MinTimeBetweenResolves
    HostAddressVec ips_checked;
    BOOST_FOREACH( const HostAddress &addr, new_ips )
    {
        if ( addr.get_ttl().get_value() < MinTimeBetweenResolves )
        {
            GlobalLogger.info() << "DnsCache: Correcting TTL of IP for "
                << key.first << " from " << addr.get_ttl().get_value() << "s to "
                << MinTimeBetweenResolves << "s because was too short";
            ips_checked.push_back( HostAddress( addr.get_ip(),
                                                MinTimeBetweenResolves) );
        }
        else
            ips_checked.push_back(addr);
    }

    // write IPs into one log line
    stringstream log_temp;
    log_temp << "DnsCache: update IPs for " << key.first << " to "
             << ips_checked.size() << "-list: ";
    BOOST_FOREACH( const HostAddress &ip, ips_checked )
        log_temp << ip.get_ip() << ", ";
    GlobalLogger.notice() << log_temp.str();

    IpCache[key] = ips_checked;
    HasChanged = true;
}


/*
 * warn if hostname is empty and remove trailing dot
 */
cname_map_key_type DnsCache::key_for_cname(const std::string &hostname) const
{
    if (hostname.empty())
    {
        GlobalLogger.info() << "DnsCache: empty host!";
        return "";
    }

    // check whether last character is a dot
    if (hostname.rfind('.') == hostname.length()-1)
        return hostname.substr(0, hostname.length()-1);
    else
        return hostname;
}


void DnsCache::update(const std::string &hostname,
                      const Cname &cname)
{
    // check for valid input arguments
    cname_map_key_type key = key_for_cname(hostname);
    if ( key.empty() )
        return;

    // ensure that there is never IP and CNAME for the same host
    int n_ips = get_ips(hostname, DNS_IPv4).size()
              + get_ips(hostname, DNS_IPv6).size();
    if ( n_ips > 0 )
    {
        GlobalLogger.info() << "DnsCache: Saving IPs for " << key
            << " removes CNAME to " << get_cname(hostname).Host << "!";
        GlobalLogger.info() << "DnsCache: Saving CNAME for " << key
            << " removes " <<  n_ips << " IPs for same host!";
        update(hostname, DNS_IPv4, HostAddressVec());
        update(hostname, DNS_IPv6, HostAddressVec());
    }

    // remove possible trailing dot from cname's target host
    Cname to_save = Cname(key_for_cname(cname.Host),  // implicit cast to string
                          cname.Ttl);

    // ensure min ttl of MinTimeBetweenResolves
    if ( to_save.Ttl.get_value() < MinTimeBetweenResolves )
    {
        GlobalLogger.info() << "DnsCache: Correcting TTL of CNAME of "
            << key << " from " << to_save.Ttl.get_value() << "s to "
            << MinTimeBetweenResolves << "s because was too short";
        to_save.Ttl = TimeToLive(MinTimeBetweenResolves);
    }

    GlobalLogger.notice() << "DnsCache: update CNAME for " << key
                          << " to " << to_save.Host;
    CnameCache[key] = to_save;
    HasChanged = true;
}


// -----------------------------------------------------------------------------
// RETRIEVAL
// -----------------------------------------------------------------------------

/**
 * @returns empty list if no (up to date) ips for hostname in cache
 */
HostAddressVec DnsCache::get_ips(const std::string &hostname,
                                 const DnsIpProtocol &protocol,
                                 const bool check_up_to_date)
{
    ip_map_key_type key = key_for_ips(hostname, protocol);
    HostAddressVec result = IpCache[key];
    if (check_up_to_date)
    {
        HostAddressVec result_up_to_date;
        uint32_t threshold = static_cast<uint32_t>(
                DnsMaster::get_instance()->get_resolved_ip_ttl_threshold() );
        uint32_t updated_ttl;
        BOOST_FOREACH( const HostAddress &addr, result )
        {
            updated_ttl = addr.get_ttl().get_updated_value();
            if (updated_ttl > threshold)
                result_up_to_date.push_back(addr);
            else
                GlobalLogger.debug() << "DnsCache: do not return "
                    << addr.get_ip().to_string() << " since TTL "
                    << updated_ttl << "s is out of date (thresh="
                    << threshold << "s)";
        }
        result = result_up_to_date;
    }
    /*GlobalLogger.debug() << "DnsCache: request IPs for " << key.first
                         << " --> " << result.size() << "-list";
    BOOST_FOREACH( const HostAddress &addr, result )
        GlobalLogger.debug() << "DnsCache:    " << addr.get_ip().to_string()
                             << " (TTL " << addr.get_ttl().get_updated_value()
                             << "s)"; */
    return result;
}

/**
 * @returns empty cname if no (up to date cname) for hostname in cache
 */
Cname DnsCache::get_cname(const std::string &hostname,
                          const bool check_up_to_date)
{
    cname_map_key_type key = key_for_cname(hostname);
    Cname result_obj = CnameCache[key];
    /*GlobalLogger.debug() << "DnsCache: request CNAME for " << key
                         << " --> \"" << result_obj.Host << "\" (TTL "
                         << result_obj.Ttl.get_updated_value() << "s)";*/
    if (result_obj.Host.empty())
        return result_obj;

    else if (check_up_to_date)
    {
        if ( result_obj.Ttl.get_updated_value() > static_cast<uint32_t>(
                   DnsMaster::get_instance()->get_resolved_ip_ttl_threshold()) )
            return result_obj;
        else
        {
            GlobalLogger.debug() << "DnsCache: CNAME is out of date";
            return Cname();    // same as if there was no cname for hostname
        }
    }
    else
        return result_obj;
}

// underlying assumption in this function: for a hostname, the cache has either
// a list of IPs saved or a cname saved, but never both
HostAddressVec DnsCache::get_ips_recursive(const std::string &hostname,
                                           const DnsIpProtocol &protocol,
                                           const bool check_up_to_date)
{
    std::string current_host = hostname;
    Cname current_cname;
    HostAddressVec result = get_ips(current_host, protocol, check_up_to_date);
    int n_recursions = 0;
    uint32_t min_cname_ttl = 0xffff;   // largest possible unsigned 4-byte value
    int max_recursion_count = DnsMaster::get_instance()
                                       ->get_max_recursion_count();
    while ( result.empty() )
    {
        current_cname = get_cname(current_host, check_up_to_date);
        if (current_cname.Host.empty())
            break;   // no ips (since result.empty()) and no cname
                     // --> will return empty result

        current_host = current_cname.Host;
        if (++n_recursions >= max_recursion_count)
        {
            GlobalLogger.info() << "DnsCache: reached recursion limit of "
                << n_recursions << " in recursive IP retrieval of "
                << hostname << "!";
            break;
        }
        else
        {
            min_cname_ttl = min(min_cname_ttl,
                                current_cname.Ttl.get_updated_value());
            result = get_ips(current_host, protocol, check_up_to_date);
        }
    }

    GlobalLogger.debug() << "DnsCache: recursive IP retrieval resulted in "
                         << result.size() << "-list after " << n_recursions
                         << " recursions";

    // adjust ttl to min of ttl and min_cname_ttl
    if (n_recursions > 0)
    {
        TimeToLive cname_ttl(min_cname_ttl);

        BOOST_FOREACH( HostAddress &addr, result )
        {
            if (addr.get_ttl().get_updated_value() > min_cname_ttl)
            {
                //GlobalLogger.debug() << "DnsCache: using shorter CNAME TTL";
                addr.set_ttl(cname_ttl);
            }
        }
    }

    return result;
}

/**
 * from a list of CNAMEs find the first one that is out of date or empty
 *
 * returns the hostname that is out of date or empty if all CNAMEs are
 *   up-to-date
 *
 * required in ResolverBase::get_skipper
 */
std::string DnsCache::get_first_outdated_cname(const std::string &hostname,
                                               const uint32_t ttl_thresh)
{
    std::string first_outdated = hostname;
    Cname cname;
    int n_recursions = 0;
    int max_recursion_count = DnsMaster::get_instance()
                                       ->get_max_recursion_count();
    while (true)
    {
        if (++n_recursions >= max_recursion_count)
        {
            GlobalLogger.info() << "DnsCache: reached recursion limit of "
                << n_recursions << " in search of outdated CNAMEs for "
                << hostname << "!";
            return first_outdated;   // not really out of date but currently
        }                            // our best guess

        cname = get_cname(first_outdated);
        if (cname.Host.empty())
            // reached end of cname list --> everything was up-to-date
            return "";
        else if (cname.Ttl.get_updated_value() > ttl_thresh)
            // cname is up to date --> continue looking
            first_outdated = cname.Host;
        else
            // cname is out of date --> return its target
            return cname.Host;
    }
    // reach this point only if cname chain does not end with an IP
    // --> all are up-to-date
    return "";
}

std::string DnsCache::get_cname_chain_str(const std::string &hostname)
{
    std::stringstream temp;
    temp << hostname;
    std::string current_host = hostname;
    Cname current_cname;
    int n_recursions = 0;
    int max_recursion_count = DnsMaster::get_instance()
                                       ->get_max_recursion_count();
    while (true)
    {
        if (n_recursions >= max_recursion_count)
        {
            temp << "...";
            break;
        }

        current_cname = get_cname(current_host, false);
        if (current_cname.Host.empty())
            break;
        else
        {
            current_host = current_cname.Host;
            temp << "-->" << current_host;
            ++n_recursions;
        }
    }
    return temp.str();
}


// -----------------------------------------------------------------------------
// OTHER
// -----------------------------------------------------------------------------
void DnsCache::debug_print() const
{
    GlobalLogger.debug() << "DnsCache: IP Cache contents:";
    stringstream log_temp;
    BOOST_FOREACH( const ip_map_type::value_type &key_and_ip, IpCache )
    {
        // write IPs into one log line
        log_temp.str("");
        log_temp << "DnsCache: " << key_and_ip.first.first << ": \t "
                 << key_and_ip.second.size() << "-list ";
        BOOST_FOREACH( const HostAddress &ip, key_and_ip.second )
            log_temp << ip.get_ip() << "+" << ip.get_ttl().get_updated_value()
                     << "s, ";
        GlobalLogger.debug() << log_temp.str();
    }

    GlobalLogger.debug() << "DnsCache: CNAME Cache contents:";
    BOOST_FOREACH( const cname_map_type::value_type &key_and_cname, CnameCache )
        GlobalLogger.debug() << "DnsCache: " << key_and_cname.first << ": \t "
                             << key_and_cname.second.Host << "+"
                             << key_and_cname.second.Ttl.get_updated_value()
                             << "s";
}