tshark -r botnet-capture-20110810-neris.pcap -T fields -e frame.time -e ip.src -e ip.dst -e ip.len -e ip.proto -e tcp.srcport -e tcp.dstport -e eth.src -e eth.dst -e udp.srcport -e udp.dstport -e udp.length -e icmp.type -e icmp.code -e icmp.seq -e dns.qry.name -e dns.qry.type -e dns.a -c 100000 -E separator=, > extracted_data.csv

tshark -r all_botnet_captures.pcap -T fields -e frame.time -e ip.src -e ip.dst -e ip.len -e ip.proto -e tcp.srcport -e tcp.dstport -e eth.src -e eth.dst -e udp.srcport -e udp.dstport -e udp.length -e icmp.type -e icmp.code -e icmp.seq -e dns.qry.name -e dns.qry.type -e dns.a -E separator=, > all_extracted_data.csv

mergecap -w all_botnet_captures.pcap botnet-capture-20110810-neris.pcap botnet-capture-20110811-neris.pcap botnet-capture-20110812-rbot.pcap botnet-capture-20110815-fast-flux.pcap botnet-capture-20110816-donbot.pcap botnet-capture-20110816-sogou.pcap botnet-capture-20110815-rbot-dos.pcap botnet-capture-20110816-qvod.pcap botnet-capture-20110815-fast-flux-2.pcap botnet-capture-20110819-bot.pcap botnet-capture-20110817-bot.pcap

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import re
from datetime import datetime

# Define constants and mappings
PROTOCOL_MAP = {
    1: 'ICMP',  # ICMP (Internet Control Message Protocol)
    6: 'TCP',   # TCP (Transmission Control Protocol)
    17: 'UDP',  # UDP (User Datagram Protocol)
    89: 'ICMPv6',  # ICMPv6 (Internet Control Message Protocol v6)
    2: 'IGMP',  # IGMP (Internet Group Management Protocol)
    41: 'IPv6',  # IPv6 (Internet Protocol version 6)
    50: 'ESP',  # ESP (Encapsulating Security Payload)
    51: 'AH',   # AH (Authentication Header)
    58: 'ICMPv6',  # ICMPv6 (Internet Control Message Protocol v6)
    0: 'Unknown',  # Unknown Protocol
    3: 'ARP',   # ARP (Address Resolution Protocol)
    132: 'SCTP',  # SCTP (Stream Control Transmission Protocol)
    33: 'DCCP',  # DCCP (Datagram Congestion Control Protocol)
    115: 'L2TP',  # L2TP (Layer 2 Tunneling Protocol)
    47: 'GRE',   # GRE (Generic Routing Encapsulation)
    88: 'EIGRP',  # EIGRP (Enhanced Interior Gateway Routing Protocol)
    89: 'OSPF',  # OSPF (Open Shortest Path First)
    39: 'LDP',   # LDP (Label Distribution Protocol)
    25: 'SMTP',  # SMTP (Simple Mail Transfer Protocol)
    53: 'DNS',   # DNS (Domain Name System)
    80: 'HTTP',  # HTTP (Hypertext Transfer Protocol)
    443: 'HTTPS',  # HTTPS (Hypertext Transfer Protocol Secure)
    21: 'FTP',   # FTP (File Transfer Protocol)
    23: 'Telnet',  # Telnet (Telecommunication Network)
    22: 'SSH',   # SSH (Secure Shell)
    123: 'NTP',   # NTP (Network Time Protocol)
}

PORT_PROTOCOL_MAP = {
    80: 'HTTP',           # HTTP (Hypertext Transfer Protocol)
    443: 'HTTPS',         # HTTPS (Hypertext Transfer Protocol Secure)
    21: 'FTP',            # FTP (File Transfer Protocol)
    22: 'SSH',            # SSH (Secure Shell)
    23: 'Telnet',         # Telnet (Telecommunication Network)
    25: 'SMTP',           # SMTP (Simple Mail Transfer Protocol)
    53: 'DNS',            # DNS (Domain Name System)
    110: 'POP3',          # POP3 (Post Office Protocol v3)
    143: 'IMAP',          # IMAP (Internet Message Access Protocol)
    3306: 'MySQL',        # MySQL (MySQL Database Service)
    8080: 'HTTP-Alt',     # HTTP Alternate (HTTP Alternative)
    8081: 'HTTP-Alt-2',   # HTTP Alternate 2 (commonly used for dev/test environments)
    67: 'DHCP Server',    # DHCP (Dynamic Host Configuration Protocol - Server)
    68: 'DHCP Client',    # DHCP (Dynamic Host Configuration Protocol - Client)
    69: 'TFTP',           # TFTP (Trivial File Transfer Protocol)
    161: 'SNMP',          # SNMP (Simple Network Management Protocol)
    162: 'SNMP Trap',     # SNMP (SNMP Trap for alerts)
    514: 'Syslog',        # Syslog (System Log Protocol)
    1433: 'MS SQL',       # MS SQL Server (Microsoft SQL Server)
    1434: 'MS SQL',       # MS SQL Server (Microsoft SQL Server - Browser Service)
    119: 'NNTP',          # NNTP (Network News Transfer Protocol)
    3389: 'RDP',          # RDP (Remote Desktop Protocol)
    465: 'SMTPS',         # SMTPS (SMTP Secure over SSL/TLS)
    993: 'IMAPS',         # IMAPS (IMAP Secure over SSL/TLS)
    995: 'POP3S',         # POP3S (POP3 Secure over SSL/TLS)
    1080: 'SOCKS Proxy',  # SOCKS Proxy
    2525: 'SMTP-Alt',     # SMTP Alternate (commonly used in email servers)
    5432: 'PostgreSQL',   # PostgreSQL (PostgreSQL Database Service)
    6379: 'Redis',        # Redis (Redis Key-Value Store)
    5900: 'VNC',          # VNC (Virtual Network Computing)
    6660: 'IRC',          # IRC (Internet Relay Chat)
    6661: 'IRC',          # IRC (Internet Relay Chat)
    6662: 'IRC',          # IRC (Internet Relay Chat)
    6663: 'IRC',          # IRC (Internet Relay Chat)
    6664: 'IRC',          # IRC (Internet Relay Chat)
    6665: 'IRC',          # IRC (Internet Relay Chat)
    6666: 'IRC',          # IRC (Internet Relay Chat)
    6667: 'IRC',          # IRC (Internet Relay Chat)
    6668: 'IRC',          # IRC (Internet Relay Chat)
    6669: 'IRC',          # IRC (Internet Relay Chat)
}


# Read the CSV file
def load_data(file_path):
    try:
        df = pd.read_csv(file_path, sep=',', engine='python', skip_blank_lines=True, on_bad_lines='skip',
                         names=['Timestamp', 'Source IP', 'Destination IP', 'Total Length', 'Protocol',
                                'TCP Source Port', 'TCP Destination Port', 'Ethernet Source', 'Ethernet Destination',
                                'UDP Source Port', 'UDP Destination Port', 'UDP Length', 'ICMP Type', 'ICMP Code',
                                'ICMP Seq', 'DNS Query Name', 'DNS Query Type', 'DNS A'])
        print("File loaded successfully.")
        return df
    except Exception as e:
        print(f"Error loading file: {e}")
        return None

# Clean and parse the 'Timestamp' column
def clean_and_parse_timestamp(timestamp):
    if isinstance(timestamp, str):
        if re.match(r'^\w{3} \d{2}$', timestamp):  # Matches 'Aug 10'
            timestamp = '2011 ' + timestamp + ' 00:00:00'  # Add default time if missing
        timestamp = re.sub(r"\s+[A-Za-z\s]+$", "", timestamp)  # Remove timezone info

        try:
            return pd.to_datetime(timestamp, errors='coerce')
        except Exception as e:
            print(f"Error parsing timestamp: {timestamp} -> {e}")
            return pd.NaT
    return pd.NaT

# Feature engineering: Adding useful time-based features
def add_time_features(df):
    df['Hour'] = df['Timestamp'].dt.hour
    df['DayOfWeek'] = df['Timestamp'].dt.dayofweek
    df['Weekday'] = df['Timestamp'].dt.weekday
    return df

# Clean 'Total Length' and fill missing values
def clean_total_length(df):
    df['Total Length'] = pd.to_numeric(df['Total Length'], errors='coerce')
    df['Total Length'] = df['Total Length'].fillna(df['Total Length'].median())
    return df

# Clean 'Source IP' and 'Destination IP'
def clean_ip_columns(df):
    df['Source IP'] = df['Source IP'].fillna('Unknown')
    df['Destination IP'] = df['Destination IP'].fillna('Unknown')
    return df

# Map 'Protocol' values using predefined map
def map_protocols(df):
    df['Protocol'] = df['Protocol'].map(PROTOCOL_MAP).fillna('Unknown')
    return df

# Apply port-based protocol mapping
def map_ports_to_protocol(df):
    def apply_port_mapping(row):
        if row['Protocol'] in ['TCP', 'UDP']:
            src_port = row['TCP Source Port'] if row['Protocol'] == 'TCP' else row['UDP Source Port']
            dest_port = row['TCP Destination Port'] if row['Protocol'] == 'TCP' else row['UDP Destination Port']
            if src_port in PORT_PROTOCOL_MAP:
                row['Protocol'] = PORT_PROTOCOL_MAP[src_port]
            elif dest_port in PORT_PROTOCOL_MAP:
                row['Protocol'] = PORT_PROTOCOL_MAP[dest_port]
        return row
    return df.apply(apply_port_mapping, axis=1)

# Clean the dataframe and prepare for analysis
def preprocess_data(df):
    df['Timestamp'] = df['Timestamp'].apply(clean_and_parse_timestamp)
    df = add_time_features(df)
    df = clean_total_length(df)
    df = clean_ip_columns(df)
    df = map_protocols(df)
    df = map_ports_to_protocol(df)
    return df

# Descriptive statistics for 'Total Length'
def total_length_stats(df):
    stats = df['Total Length'].describe()
    print("\nDescriptive Statistics for Total Length:")
    print(stats)
    return stats

# Outlier detection using IQR
def detect_outliers(df):
    Q1 = df['Total Length'].quantile(0.25)
    Q3 = df['Total Length'].quantile(0.75)
    IQR = Q3 - Q1
    lower_bound = Q1 - 1.5 * IQR
    upper_bound = Q3 + 1.5 * IQR
    outliers = df[(df['Total Length'] < lower_bound) | (df['Total Length'] > upper_bound)]
    print(f"\nNumber of outliers: {len(outliers)}")
    print(outliers.head(10))
    return outliers, lower_bound, upper_bound

# Plot distributions
def plot_total_length_distribution(df):
    plt.figure(figsize=(10, 6))
    plt.subplot(1, 2, 1)
    sns.histplot(df['Total Length'], bins=50, kde=True, color='blue')
    plt.title('Total Length Distribution')

    plt.subplot(1, 2, 2)
    sns.boxplot(x=df['Total Length'], color='orange')
    plt.title('Total Length Boxplot')
    plt.xlabel('Total Length')
    plt.ylabel('Frequency')
    plt.tight_layout()
    plt.show()

    # Output of the Total Length Distribution and Boxplot
    print("\nVisual Output for Total Length Distribution and Boxplot:")
    print(f"- The histogram shows the overall distribution of total length values.")
    print(f"- The boxplot highlights the spread and identifies potential outliers in the data.")

# Hourly Total Length Statistics
def plot_hourly_total_length(df):
    plt.figure(figsize=(10, 6))
    sns.scatterplot(x='Hour', y='Total Length', data=df, color='green')
    plt.title('Total Length vs. Hour of the Day')
    plt.xlabel('Hour of the Day')
    plt.ylabel('Total Length')
    plt.show()

    # Output of Total Length vs. Hour of Day
    print("\nVisual Output for Total Length vs. Hour of the Day:")
    hourly_stats = df.groupby('Hour')['Total Length'].mean()
    print(f"- The scatter plot shows the relationship between total length and the hour of the day.")
    print(f"- Here are the average total lengths per hour:\n{hourly_stats}")

# Frequency of each protocol
def plot_protocol_frequency(df):
    protocol_counts = df['Protocol'].value_counts()
    plt.figure(figsize=(10, 6))
    sns.barplot(x=protocol_counts.index, y=protocol_counts.values, palette='viridis', hue=protocol_counts.index)
    plt.title('Frequency Distribution of Protocols')
    plt.xticks(rotation=90)
    plt.xlabel('Protocol')
    plt.ylabel('Frequency')
    plt.show()

    # Output of Protocol Frequency
    print("\nVisual Output for Protocol Frequency Distribution:")
    print(f"- The bar chart shows the frequency distribution of protocols.")
    print(f"- Here are the counts for each protocol:\n{protocol_counts}")

# Traffic by weekday
def plot_traffic_by_weekday(df):
    weekday_traffic = df.groupby('DayOfWeek')['Total Length'].sum().reset_index()
    plt.figure(figsize=(10, 6))
    sns.barplot(x=weekday_traffic['DayOfWeek'], y=weekday_traffic['Total Length'],
                palette='viridis', hue=weekday_traffic['DayOfWeek'])
    plt.title('Traffic Volume by Day of Week')
    plt.xlabel('Day of Week')
    plt.ylabel('Total Length')
    plt.xticks(ticks=range(7), labels=["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"])
    plt.show()

    # Output of Traffic by Weekday
    print("\nVisual Output for Traffic Volume by Day of Week:")
    print(f"- The bar chart shows the total traffic volume for each day of the week.")
    print(f"- Traffic patterns per weekday:\n{weekday_traffic}")

# Traffic volume by hour
def plot_traffic_by_hour(df):
    hourly_traffic = df.groupby('Hour')['Total Length'].sum().reset_index()
    plt.figure(figsize=(10, 6))
    sns.lineplot(x=hourly_traffic['Hour'], y=hourly_traffic['Total Length'], color='purple')
    plt.title('Traffic Volume by Hour of the Day')
    plt.xlabel('Hour of the Day')
    plt.ylabel('Total Length')
    plt.show()

    # Output of Traffic Volume by Hour
    print("\nVisual Output for Traffic Volume by Hour of the Day:")
    print(f"- The line plot shows the total traffic volume aggregated by each hour.")
    print(f"- Hourly traffic summary:\n{hourly_traffic}")

# Main function to run all steps
def main(file_path):
    df = load_data(file_path)
    if df is not None:
        df = preprocess_data(df)

        # Descriptive statistics and outlier detection
        total_length_stats(df)
        outliers, lower_bound, upper_bound = detect_outliers(df)

        # Plotting and visual outputs
        plot_total_length_distribution(df)
        plot_hourly_total_length(df)
        plot_protocol_frequency(df)
        plot_traffic_by_weekday(df)
        plot_traffic_by_hour(df)

# Run the analysis with the given file path
file_path = 'Documents/pcap_files/all_extracted_data.csv'  # Update with your actual file path
main(file_path)