can now interpret pcaps from tcpdump; created own pcap.h and pcap.cpp because is...
authorChristian Herdtweck <christian.herdtweck@intra2net.com>
Thu, 19 Mar 2015 17:25:18 +0000 (18:25 +0100)
committerChristian Herdtweck <christian.herdtweck@intra2net.com>
Thu, 19 Mar 2015 17:25:18 +0000 (18:25 +0100)
* icmp/icmppacketfactory
* tools/feed_packet_data
* tests/test_icmppacket

pcaps from tcpdump have network_type = 1, so include a interface struct in packet data before actual data.
Our own dumps were network_type=101 and did not include that interface info

src/CMakeLists.txt
src/icmp/icmppacketfactory.cpp
src/icmp/icmppacketfactory.h
src/tools/feed_packet_data.cpp
src/tools/pcap.cpp [new file with mode: 0644]
src/tools/pcap.h [new file with mode: 0644]
test/CMakeLists.test_icmppacket.txt
test/test_icmppacket.cpp

index 9bf618b..2003089 100644 (file)
@@ -76,6 +76,7 @@ set(SOURCES
     host/pingprotocol.cpp
     host/pingrotate.cpp
     host/pingscheduler.cpp
+    tools/pcap.cpp
     icmp/icmpdata.cpp
     icmp/icmpdata_pingfailreply.cpp
     icmp/icmpechodata.cpp
@@ -133,7 +134,7 @@ install(TARGETS ${TARGET} DESTINATION bin)
 add_library(lib_for_tools STATIC ${SOURCES})
 
 # now create an executable for tools/feed_packet_data that depends on lib_for_tools
-set(feed_packet_data_SOURCES tools/feed_packet_data.cpp)
+set(feed_packet_data_SOURCES tools/pcap.cpp tools/feed_packet_data.cpp)
 add_executable(feed_packet_data ${feed_packet_data_SOURCES})
 target_link_libraries(
     feed_packet_data
index 4cd9839..bcfa74b 100644 (file)
 #include "icmp/icmpheader.h"
 #include "icmp/icmptype.h"
 #include "icmp/icmpechodata.h"
+#include "tools/pcap.h"
 
 using namespace std;
 using boost::asio::ip::icmp;
 using I2n::Logger::GlobalLogger;
 
-//-----------------------------------------------------------------------------
-// Dump of data packets to pcap files
-//-----------------------------------------------------------------------------
-#pragma pack(push, 1) // exact fit -- no padding of data in structs
-
-// pcap header for dumping of packet data
-// (http://wiki.wireshark.org/Development/LibpcapFileFormat)
-typedef struct pcap_hdr_s {
-        uint32_t magic_number;   /* magic number */
-        uint16_t version_major;  /* major version number */
-        uint16_t version_minor;  /* minor version number */
-        int32_t  thiszone;       /* GMT to local correction */
-        uint32_t sigfigs;        /* accuracy of timestamps */
-        uint32_t snaplen;        /* max length of captured packets, in octets */
-        uint32_t network;        /* data link type */
-} pcap_hdr_t;
-
-typedef struct pcaprec_hdr_s {
-        uint32_t ts_sec;         /* timestamp seconds */
-        uint32_t ts_usec;        /* timestamp microseconds */
-        uint32_t incl_len;       /* number of octets of packet saved in file */
-        uint32_t orig_len;       /* actual length of packet */
-} pcaprec_hdr_t;
-
-
 void dump_packet(const std::string &data)
 {
-    std::size_t data_size = data.size();
-
     // create unique file name
     std::stringstream temp_name;
     temp_name << "/datastore/pingcheck.broken/icmp_";
@@ -93,40 +67,18 @@ void dump_packet(const std::string &data)
         return;
     }
 
-    // create a pcap header
-    pcap_hdr_t pcap_hdr;
-    pcap_hdr.magic_number = 0xa1b2c3d4;
-    pcap_hdr.version_major = 2;
-    pcap_hdr.version_minor = 4;
-    pcap_hdr.thiszone = 0;
-    pcap_hdr.sigfigs = 0;
-    pcap_hdr.snaplen = 40;
-    pcap_hdr.network = 101;   // LINKTYPE_RAW, according to
-                              // http://www.tcpdump.org/linktypes.html
-
-    // create a record (packet) header
-    pcaprec_hdr_t pcaprec_hdr;
-    pcaprec_hdr.ts_sec = static_cast<uint32_t>(capture_time);
-    pcaprec_hdr.ts_usec = 0;    // do not care about that precision
-    pcaprec_hdr.incl_len = data_size;
-    pcaprec_hdr.orig_len = data_size;
-
     // create file pointer for file descriptor
     FILE *fp = fdopen(fd, "w");
 
-    // write data
-    fwrite(&pcap_hdr, sizeof(pcap_hdr_t), 1, fp);
-    fwrite(&pcaprec_hdr, sizeof(pcaprec_hdr_t), 1, fp);
-    fwrite(data.c_str(), sizeof(char), data_size, fp);
+    // dump data
+    write_pcap_packet_data(data, fp, capture_time);
 
-    // done -- clean up
+    // clean up
     fclose(fp);
     close(fd);
     GlobalLogger.debug() << "Dumped a copy of the data into "
                          << secure_filename.get() << endl;
 }
-#pragma pack(pop) // restore old value
-
 //-----------------------------------------------------------------------------
 // IcmpPacketFactory
 //-----------------------------------------------------------------------------
index f954ed3..dd1a07b 100644 (file)
@@ -43,6 +43,9 @@ void dump_packet(const std::string &data);
 /**
  * @brief Class which constructs ICMP Packets, hiding from the class users the
  * underlying details on how to build each packet.
+ *
+ * @param dump_mode: 0 to never dump, 1 to dump in case of parsing trouble,
+ *                   2 to always dump
  */
 class IcmpPacketFactory
 {
index e5a8c3a..d11ab10 100644 (file)
@@ -27,6 +27,7 @@ on this file might be covered by the GNU General Public License.
 
 #include "icmp/icmppacketfactory.h"
 #include "tcp/tcpsegmentfactory.h"
+#include "tools/pcap.h"
 
 using I2n::Logger::GlobalLogger;
 namespace po = boost::program_options;
@@ -37,24 +38,10 @@ namespace po = boost::program_options;
 const bool default_is_icmp = true;
 const bool default_is_v4 = true;
 
-//const uint32_t pcap_magic_number = 0xa1b2c3d4;
-typedef unsigned char byte;
-const byte pcap_magic_bytes[] = {0xa1, 0xb2, 0xc3, 0xd4};
-
-// pcap file header is 5 x uint32 + 2 x uint16 --> 24 bytes
-const std::streamsize pcap_file_header_size = 24;
-
-// pcap file header is 4 x uint32 --> 16 bytes
-const std::streamsize pcap_packet_header_size = 16;
-
-
 //------------------------------------------------------------------------------
 // function declarations
 void init_logger();
 void increase_verbosity();
-bool check_for_pcap_header(std::istream &input_stream);
-void consume_pcap_file_header(std::istream &input_stream);
-void consume_pcap_packet_header(std::istream &input_stream);
 int read_packets(std::istream &input_stream, const bool, const bool, const int);
 
 //------------------------------------------------------------------------------
@@ -74,51 +61,6 @@ void increase_verbosity()
     GlobalLogger.info() << "Increased verbosity";
 }
 
-// determine whether is raw data or pcap by peeking at first 4 bytes
-bool check_for_pcap_header(std::istream &input_stream)
-{
-    int peek_idx;
-    bool is_pcap = true;
-    for (peek_idx=0; peek_idx<4; ++peek_idx)
-    {
-        int val = input_stream.get();
-        //GlobalLogger.debug() << "Peeked value " << std::hex << val;
-        if (val != pcap_magic_bytes[3-peek_idx])
-        {
-            is_pcap = false;
-            break;
-        }
-    }
-    for (int back_idx=peek_idx; back_idx > 0; --back_idx)
-        input_stream.unget();
-    if (is_pcap)
-        GlobalLogger.info() << "File is pcap";
-    else
-        GlobalLogger.info() << "File is not pcap";
-    return is_pcap;
-}
-
-void consume_pcap_file_header(std::istream &input_stream)
-{
-    GlobalLogger.debug() << "Consuming pcap file header";
-    char *buf = new char[pcap_file_header_size];
-    input_stream.read(buf, pcap_file_header_size);
-    // could do some checking whether this data actually looks like a
-    // pcap header --> see ../icmp/icmppacketfactory.cpp
-    delete[] buf;
-}
-
-void consume_pcap_packet_header(std::istream &input_stream)
-{
-    GlobalLogger.debug() << "Consuming pcap packet header";
-    char *buf = new char[pcap_packet_header_size];
-    input_stream.read(buf, pcap_packet_header_size);
-    // could do some checking whether this data actually looks like a
-    // pcap header --> see ../icmp/icmppacketfactory.cpp
-    delete[] buf;
-}
-
-
 /**
  * @brief read packets from given input stream
  *
@@ -130,10 +72,19 @@ int read_packets( std::istream &input_stream, const bool is_icmp,
 {
     // peek at start of stream to see if there is a pcap header
     bool is_pcap = check_for_pcap_header(input_stream);
+    if (is_pcap)
+        GlobalLogger.info() << "File is pcap";
+    else
+        GlobalLogger.info() << "File is not pcap";
+
     int next_val;
     int packet_count = 0;
+    uint32_t packet_type = 0;
     if (is_pcap)
-        consume_pcap_file_header(input_stream);
+    {
+        packet_type = consume_pcap_file_header(input_stream);
+        GlobalLogger.debug() << "Packet type from header is " << packet_type;
+    }
     if ( !input_stream )
     {
         GlobalLogger.notice() << "Failure consuming pcap file header!";
@@ -146,12 +97,24 @@ int read_packets( std::istream &input_stream, const bool is_icmp,
         if (is_pcap)
         {
             // assume that there is a packet header if stream had pcap header
+            GlobalLogger.debug() << "Consume packet header";
             consume_pcap_packet_header(input_stream);
             if ( !input_stream )
             {
                 GlobalLogger.notice() <<"Failure consuming pcap packet header!";
                 return 4;
             }
+
+            if (packet_type == 1)
+            {
+                GlobalLogger.debug() << "Consume ethernet header";
+                consume_pcap_ethernet_header(input_stream);
+                if ( !input_stream )
+                {
+                    GlobalLogger.notice() <<"Failure consuming pcap packet header!";
+                    return 5;
+                }
+            }
         }
 
         // feed data to right factory
@@ -181,7 +144,7 @@ int read_packets( std::istream &input_stream, const bool is_icmp,
             else
             {
                 GlobalLogger.notice() << "ICMP packet creation failed";
-                return 5;
+                return 6;
             }
         }
         else   // is tcp
@@ -209,14 +172,14 @@ int read_packets( std::istream &input_stream, const bool is_icmp,
             else
             {
                 GlobalLogger.notice() << "TCP segment creation failed";
-                return 5;
+                return 6;
             }
         }
 
         if ( !input_stream )
         {
             GlobalLogger.notice() << "Stream not good after reading!";
-            return 6;
+            return 7;
         }
 
         // if reached this point, created a packet, otherwise will have
@@ -230,7 +193,27 @@ int read_packets( std::istream &input_stream, const bool is_icmp,
             GlobalLogger.info() << "Reached end of stream";
             break;
         }
-        input_stream.unget();
+        GlobalLogger.debug() << "Stream continues with value " << std::showbase
+                             << std::hex << next_val;
+
+        if (is_pcap && packet_type == 1 && next_val == 0)
+        {
+            GlobalLogger.debug() << "Consume 0s added for alignment";
+            while (input_stream.good() && next_val == 0)
+                next_val = input_stream.get();
+
+            // if stream is still good, then got a non-0 value from it
+            // --> put it back
+            if (input_stream.good())
+                input_stream.unget();
+            else
+            {
+                GlobalLogger.info() << "Reached end of stream";
+                break;
+            }
+        }
+        else
+            input_stream.unget();
     } //eo: while (input_stream)
 
     return (-1) * packet_count;
diff --git a/src/tools/pcap.cpp b/src/tools/pcap.cpp
new file mode 100644 (file)
index 0000000..d7e1dd3
--- /dev/null
@@ -0,0 +1,145 @@
+/*
+ The software in this package is distributed under the GNU General
+ Public License version 2 (with a special exception described below).
+
+ A copy of GNU General Public License (GPL) is included in this distribution,
+ in the file COPYING.GPL.
+
+ As a special exception, if other files instantiate templates or use macros
+ or inline functions from this file, or you compile this file and link it
+ with other works to produce a work based on this file, this file
+ does not by itself cause the resulting work to be covered
+ by the GNU General Public License.
+
+ However the source code for this file must still be made available
+ in accordance with section (3) of the GNU General Public License.
+
+ This exception does not invalidate any other reasons why a work based
+ on this file might be covered by the GNU General Public License.
+
+ Christian Herdtweck, Intra2net AG 2015
+ */
+
+#include "tools/pcap.h"
+#include <cstdio>
+
+//-----------------------------------------------------------------------------
+// Dump of data packets to pcap files
+//-----------------------------------------------------------------------------
+#pragma pack(push, 1) // exact fit -- no padding of data in structs
+
+void write_pcap_packet_data(const std::string &data,
+                            FILE *fp,
+                            const time_t &capture_time)
+{
+    std::size_t data_size = data.size();
+
+    // create a pcap header
+    pcapfile_hdr_t pcap_hdr;
+    pcap_hdr.magic_number = 0xa1b2c3d4;
+    pcap_hdr.version_major = 2;
+    pcap_hdr.version_minor = 4;
+    pcap_hdr.thiszone = 0;
+    pcap_hdr.sigfigs = 0;
+    pcap_hdr.snaplen = data_size;
+    pcap_hdr.network = 101;   // LINKTYPE_RAW, according to
+                              // http://www.tcpdump.org/linktypes.html
+
+    // create a record (packet) header
+    pcaprec_hdr_t pcaprec_hdr;
+    pcaprec_hdr.ts_sec = static_cast<uint32_t>(capture_time);
+    pcaprec_hdr.ts_usec = 0;    // do not care about that precision
+    pcaprec_hdr.incl_len = data_size;
+    pcaprec_hdr.orig_len = data_size;
+
+    // write data
+    fwrite(&pcap_hdr, sizeof(pcapfile_hdr_t), 1, fp);
+    fwrite(&pcaprec_hdr, sizeof(pcaprec_hdr_t), 1, fp);
+    fwrite(data.c_str(), sizeof(char), data_size, fp);
+}
+
+//const uint32_t pcap_magic_number = 0xa1b2c3d4;
+const uint8_t pcap_magic_bytes[] = {0xa1, 0xb2, 0xc3, 0xd4};
+
+// pcap file header is 5 x uint32 + 2 x uint16 --> 24 bytes
+const std::streamsize pcap_file_header_size = sizeof(pcapfile_hdr_t);
+
+// pcap file header is 4 x uint32 --> 16 bytes
+const std::streamsize pcap_packet_header_size = sizeof(pcaprec_hdr_t);
+
+// pcap ethernet header is 2 x 6 byte + 2 byte --> 14 bytes
+const std::streamsize pcap_ethernet_header_size = sizeof(pcapeth_hdr_t);
+
+// determine whether is raw data or pcap by peeking at first 4 bytes
+bool check_for_pcap_header(std::istream &input_stream)
+{
+    int peek_idx;
+    bool is_pcap = true;
+    for (peek_idx=0; peek_idx<4; ++peek_idx)
+    {
+        int val = input_stream.get();
+        //GlobalLogger.debug() << "Peeked value " << std::hex << val;
+        if (val != pcap_magic_bytes[3-peek_idx])
+        {
+            is_pcap = false;
+            break;
+        }
+    }
+    for (int back_idx=peek_idx; back_idx > 0; --back_idx)
+        input_stream.unget();
+    return is_pcap;
+}
+
+uint32_t consume_pcap_file_header(std::istream &input_stream)
+{
+    char *buf = new char[pcap_file_header_size];
+    input_stream.read(buf, pcap_file_header_size);
+
+    // interpret as pcap header
+    pcapfile_hdr_t *header = (pcapfile_hdr_t*) reinterpret_cast<uint8_t*>(buf);
+    // could do some checking whether this data actually looks like a valid
+    // pcap header
+
+    uint32_t result = header->network;
+    delete[] buf;
+    return result;
+}
+
+void consume_pcap_packet_header(std::istream &input_stream)
+{
+    char *buf = new char[pcap_packet_header_size];
+    input_stream.read(buf, pcap_packet_header_size);
+    // could do some checking whether this data actually looks like a
+    // pcap header --> see ../icmp/icmppacketfactory.cpp
+    delete[] buf;
+}
+
+
+void consume_pcap_ethernet_header(std::istream &input_stream)
+{
+    char *buf = new char[pcap_ethernet_header_size];
+    input_stream.read(buf, pcap_ethernet_header_size);
+    // could do some checking whether this data actually looks like a
+    // pcap header --> see ../icmp/icmppacketfactory.cpp
+    delete[] buf;
+}
+
+bool consume_single_packet_pcap(std::istream &input_stream)
+{
+    bool is_pcap = check_for_pcap_header(input_stream);
+    if (!is_pcap)
+        return false;
+
+    uint32_t packet_type = consume_pcap_file_header(input_stream);
+    consume_pcap_packet_header(input_stream);
+    if (packet_type == 1)
+        consume_pcap_ethernet_header(input_stream);
+
+    return true;
+}
+
+
+#pragma pack(pop) // restore old value
+
+// (created using vim -- the world's best text editor)
+
diff --git a/src/tools/pcap.h b/src/tools/pcap.h
new file mode 100644 (file)
index 0000000..0d3a460
--- /dev/null
@@ -0,0 +1,79 @@
+/*
+ The software in this package is distributed under the GNU General
+ Public License version 2 (with a special exception described below).
+
+ A copy of GNU General Public License (GPL) is included in this distribution,
+ in the file COPYING.GPL.
+
+ As a special exception, if other files instantiate templates or use macros
+ or inline functions from this file, or you compile this file and link it
+ with other works to produce a work based on this file, this file
+ does not by itself cause the resulting work to be covered
+ by the GNU General Public License.
+
+ However the source code for this file must still be made available
+ in accordance with section (3) of the GNU General Public License.
+
+ This exception does not invalidate any other reasons why a work based
+ on this file might be covered by the GNU General Public License.
+
+ Christian Herdtweck, Intra2net AG 2015
+ */
+
+#ifndef PCAP_H
+#define PCAP_H
+
+#include <stdint.h>
+#include <istream>
+#include <ctime>
+
+#pragma pack(push, 1) // exact fit -- no padding of data in structs
+
+// pcap header for dumping of packet data
+// (http://wiki.wireshark.org/Development/LibpcapFileFormat)
+typedef struct pcapfile_hdr_s {
+        uint32_t magic_number;   /* magic number */
+        uint16_t version_major;  /* major version number */
+        uint16_t version_minor;  /* minor version number */
+        int32_t  thiszone;       /* GMT to local correction */
+        uint32_t sigfigs;        /* accuracy of timestamps */
+        uint32_t snaplen;        /* max length of captured packets, in octets */
+        uint32_t network;        /* data link type */
+} pcapfile_hdr_t;
+
+typedef struct pcaprec_hdr_s {
+        uint32_t ts_sec;         /* timestamp seconds */
+        uint32_t ts_usec;        /* timestamp microseconds */
+        uint32_t incl_len;       /* number of octets of packet saved in file */
+        uint32_t orig_len;       /* actual length of packet */
+} pcaprec_hdr_t;
+
+// not used here but in feed_packet_data and want to keep things together
+// this structure is contained in packet data if pcapfile_hdr_t.network is 1
+// adapted from http://www.tcpdump.org/pcap.html
+typedef struct pcapeth_hdr_s {
+        uint8_t source_mac_address[6];
+        uint8_t destination_mac_address[6];
+        uint16_t ether_type;
+} pcapeth_hdr_t;
+
+void write_pcap_packet_data(const std::string &data,
+                            FILE *fp,
+                            const time_t &capture_time);
+
+bool check_for_pcap_header(std::istream &input_stream);
+uint32_t consume_pcap_file_header(std::istream &input_stream);
+void consume_pcap_packet_header(std::istream &input_stream);
+void consume_pcap_ethernet_header(std::istream &input_stream);
+//void consume_pcap_padding_zeros(std::istream &input_stream);
+//  --> see feed_packet_data.cpp
+
+// returns true if is is pcap, false otherwise
+bool consume_single_packet_pcap(std::istream &input_stream);
+
+#pragma pack(pop) // restore old value
+
+#endif
+
+// (created using vim -- the world's best text editor)
+
index 822ab62..260ba6d 100644 (file)
@@ -1,6 +1,7 @@
 # compiler: creates the binaries
 add_executable(test_icmppacket
     ${CMAKE_SOURCE_DIR}/src/boost_assert_handler.cpp
+    ${CMAKE_SOURCE_DIR}/src/tools/pcap.cpp
     ${CMAKE_SOURCE_DIR}/src/host/messagepayload.cpp
     ${CMAKE_SOURCE_DIR}/src/ip/ipheader.cpp
     ${CMAKE_SOURCE_DIR}/src/ip/ipv4header.cpp
index 6266b25..5322624 100644 (file)
 
 #include "icmp/icmppacketfactory.h"
 #include "icmp/icmptype.h"
+#include "tools/pcap.h"
 
 //------------------------------------------------------------------------------
 //  helper function and consts
 //------------------------------------------------------------------------------
 
-bool check_and_consume_pcap_headers(std::ifstream &input_stream);
-IcmpPacketItem read_packet(const std::string &file_name);
-
-const uint8_t pcap_magic_bytes[] = {0xa1, 0xb2, 0xc3, 0xd4};
-const std::streamsize n_magic_bytes = 4;
-const std::streamsize pcap_file_header_size = 24;
-const std::streamsize pcap_packet_header_size = 16;
-
 const int DUMP_MODE_NO_DUMP = 0;
 
-bool check_and_consume_pcap_headers(std::ifstream &input_stream)
-{
-    int read_idx;
-    for (read_idx=0; read_idx<4; ++read_idx)
-    {
-        int val = input_stream.get();
-        if (static_cast<uint8_t>(val) != pcap_magic_bytes[3-read_idx])
-            return false;
-    }
-
-    std::streamsize to_consume = pcap_file_header_size + pcap_packet_header_size
-                                 - n_magic_bytes;
-    char *buf = new char[to_consume];
-    input_stream.read(buf, to_consume);
-    delete[] buf;
+IcmpPacketItem read_packet(const std::string &file_name);
 
-    return true;
-}
 
 IcmpPacketItem read_packet(const std::string &file_name)
 {
-    std::ifstream file_stream(file_name.c_str(), std::ios::in |
+    std::stringstream full_name;
+    full_name << DATA_DIR_STRING << "/" << file_name;
+    BOOST_TEST_MESSAGE( "Opening file " << full_name.str() );
+    std::ifstream file_stream(full_name.str().c_str(), std::ios::in |
                                            std::ios::binary);
     // could open?
     BOOST_REQUIRE( file_stream );
 
     // recognized pcap file?
-    bool is_pcap = check_and_consume_pcap_headers(file_stream);
+    bool is_pcap = consume_single_packet_pcap(file_stream);
     if ( !is_pcap )
     {
         file_stream.close();