mirror of
https://github.com/ltcptgeneral/IdealRMT-DecisionTrees.git
synced 2026-10-07 16:28:31 +00:00
Add support for combined datasets and analysis
This commit is contained in:
Jai Parera committed
1 parent
541538fcfe
commit
24fc2ed6f7
11 files changed
+1082
-5
No files matched your search
@@ -0,0 +1,2 @@
|
||||
# force LF for any shell script
|
||||
*.sh text eol=lf
|
||||
+3
-1
@@ -1,3 +1,5 @@
|
||||
data.*
|
||||
__pycache__
|
||||
*.json
|
||||
*.json
|
||||
|
||||
data/*
|
||||
@@ -2,17 +2,21 @@
|
||||
|
||||
Run `pip install -r requirements.txt`
|
||||
|
||||
Run `setup.sh`
|
||||
|
||||
# Tree Generation
|
||||
|
||||
## Download Dataset
|
||||
|
||||
Download the *September 22 2016* dataset from: https://iotanalytics.unsw.edu.au/iottraces.html#bib18tmc
|
||||
Download the *September 22 2016* dataset (or others) from: https://iotanalytics.unsw.edu.au/iottraces.html#bib18tmc
|
||||
|
||||
Rename the file as data.pcap
|
||||
Place these into the `data/tar` folder.
|
||||
|
||||
Run `extract_tars.sh` which will extract and place the `.pcap` files at the corresponding location inside `data/pcap`.
|
||||
|
||||
## Preprocessing Dataset
|
||||
|
||||
Run `ExtractDataset.ipynb`, this will take a few minutes
|
||||
Run `extract_all_datasets.py` which will extract the data from each file in `data/pcap` and turn it into the corresponding `.csv` file inside `data/processed`. This will take a few minutes per file. Combine the data under `data/csv` using `combine_csv.py`. This will overwrite `data/combined/data.csv` which you can use for the decision tree.
|
||||
|
||||
## Training
|
||||
|
||||
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
"""combined.py
|
||||
|
||||
Concatenate every CSV that matches the pattern
|
||||
data/processed/<name>/<name>.csv
|
||||
into a single file:
|
||||
data/combined/data.csv
|
||||
|
||||
The script streams each source CSV in 1‑Mio‑row chunks so memory stays low.
|
||||
Typos in the historic column names (protocl/classfication) are fixed on‑the‑fly.
|
||||
|
||||
Usage
|
||||
-----
|
||||
python combined.py
|
||||
|
||||
You can optionally supply a different root directory:
|
||||
python combined.py --root other/processed_dir --out other/combined/data.csv
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import os
|
||||
import pandas as pd
|
||||
|
||||
CHUNK = 1_000_000 # rows per read_csv chunk
|
||||
|
||||
|
||||
def fix_cols(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Rename legacy columns to canonical names."""
|
||||
return df.rename(
|
||||
columns={"protocl": "protocol", "classfication": "classification"}
|
||||
)
|
||||
|
||||
|
||||
def find_source_csvs(proc_root: Path):
|
||||
"""Yield CSV paths that exactly match processed/<name>/<name>.csv."""
|
||||
for sub in sorted(proc_root.iterdir()):
|
||||
if not sub.is_dir():
|
||||
continue
|
||||
target = sub / f"{sub.name}.csv"
|
||||
if target.exists():
|
||||
yield target
|
||||
|
||||
|
||||
def combine(proc_root: Path, out_path: Path):
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
first_write = True
|
||||
for csv_path in find_source_csvs(proc_root):
|
||||
print(f"→ adding {csv_path.relative_to(proc_root.parent)}")
|
||||
for chunk in pd.read_csv(csv_path, chunksize=CHUNK):
|
||||
chunk = fix_cols(chunk)
|
||||
chunk.to_csv(
|
||||
out_path,
|
||||
mode="w" if first_write else "a",
|
||||
header=first_write,
|
||||
index=False,
|
||||
)
|
||||
first_write = False
|
||||
print(f"✓ combined CSV written to {out_path}")
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="Combine processed CSVs into one.")
|
||||
p.add_argument("--root", default="data/processed", help="processed dir root")
|
||||
p.add_argument("--out", default="data/combined/data.csv", help="output CSV")
|
||||
args = p.parse_args()
|
||||
|
||||
combine(Path(args.root).expanduser(), Path(args.out).expanduser())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from labels import mac_to_label
|
||||
from tqdm import tqdm
|
||||
import os
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
PCAP_DIR = ROOT / "data" / "pcap"
|
||||
CSV_DIR = ROOT / "data" / "processed"
|
||||
CSV_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
BATCH = 100_000 # packets per chunk
|
||||
|
||||
from scapy.all import rdpcap
|
||||
|
||||
|
||||
def process_pcap(pcap_path: str, csv_path: str) -> None:
|
||||
all_packets = rdpcap(pcap_path)
|
||||
|
||||
print("rdpcap done", flush=True)
|
||||
results = []
|
||||
for packet in tqdm(all_packets):
|
||||
size = len(packet)
|
||||
try:
|
||||
proto = packet.proto
|
||||
except AttributeError:
|
||||
proto = 0
|
||||
try:
|
||||
sport = packet.sport
|
||||
dport = packet.dport
|
||||
except AttributeError:
|
||||
sport = 0
|
||||
dport = 0
|
||||
|
||||
proto = int(proto)
|
||||
sport = int(sport)
|
||||
dport = int(dport)
|
||||
|
||||
if "Ether" in packet:
|
||||
eth_dst = packet["Ether"].dst
|
||||
if eth_dst in mac_to_label:
|
||||
classification = mac_to_label[eth_dst]
|
||||
else:
|
||||
classification = "other"
|
||||
else:
|
||||
classification = "other"
|
||||
|
||||
metric = [proto,sport,dport,classification]
|
||||
results.append(metric)
|
||||
results = (np.array(results)).T
|
||||
|
||||
# store the features in the dataframe
|
||||
dataframe = pd.DataFrame({'protocl':results[0],'src':results[1],'dst':results[2],'classfication':results[3]})
|
||||
columns = ['protocl','src','dst','classfication']
|
||||
|
||||
# save the dataframe to the csv file, if not exsit, create one.
|
||||
if os.path.exists(csv_path):
|
||||
dataframe.to_csv(csv_path,index=False,sep=',',mode='a',columns = columns, header=False)
|
||||
else:
|
||||
dataframe.to_csv(csv_path,index=False,sep=',',columns = columns)
|
||||
|
||||
print("Done")
|
||||
|
||||
|
||||
|
||||
def main() -> None:
|
||||
for pcap in sorted(PCAP_DIR.rglob("*.pcap")):
|
||||
rel_csv = pcap.relative_to(PCAP_DIR).with_suffix(".csv")
|
||||
csv_path = CSV_DIR / rel_csv
|
||||
if csv_path.exists():
|
||||
print(f"Skip {rel_csv} (CSV exists)")
|
||||
continue
|
||||
print(f"Processing {rel_csv}")
|
||||
csv_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
process_pcap(str(pcap), str(csv_path))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
# Usage: extract_all.sh SOURCE_DIR TARGET_DIR
|
||||
# For every .tar, .tar.gz, .tgz, .tar.bz2, .tar.xz in SOURCE_DIR:
|
||||
# 1. Create TARGET_DIR/<name>/
|
||||
# 2. If TARGET_DIR/<name>/<name>.pcap already exists, skip the archive.
|
||||
# 3. Otherwise, extract the archive into its own folder.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $# -ne 2 ]]; then
|
||||
echo "Usage: $0 SOURCE_DIR TARGET_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
src_dir="$1"
|
||||
dst_dir="$2"
|
||||
mkdir -p "$dst_dir"
|
||||
|
||||
# Strip common extensions to recover the base name
|
||||
strip_ext() {
|
||||
local n="$1"
|
||||
n=${n%.tar.gz}; n=${n%.tgz}; n=${n%.tar.bz2}; n=${n%.tar.xz}; n=${n%.tar}
|
||||
echo "$n"
|
||||
}
|
||||
|
||||
shopt -s nullglob
|
||||
for archive in "$src_dir"/*.tar{,.gz,.bz2,.xz} "$src_dir"/*.tgz; do
|
||||
base=$(basename "$archive")
|
||||
name=$(strip_ext "$base")
|
||||
out_dir="$dst_dir/$name"
|
||||
key_file="$out_dir/$name.pcap"
|
||||
|
||||
if [[ -f "$key_file" ]]; then
|
||||
echo "Skipping $archive — $key_file already present"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Extracting $archive into $out_dir"
|
||||
mkdir -p "$out_dir"
|
||||
|
||||
case "$archive" in
|
||||
*.tar) tar -xf "$archive" -C "$out_dir" ;;
|
||||
*.tar.gz|*.tgz) tar -xzf "$archive" -C "$out_dir" ;;
|
||||
*.tar.bz2) tar -xjf "$archive" -C "$out_dir" ;;
|
||||
*.tar.xz) tar -xJf "$archive" -C "$out_dir" ;;
|
||||
*) echo "Unknown type: $archive" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "All archives processed."
|
||||
+2
-1
@@ -3,4 +3,5 @@ numpy
|
||||
pandas
|
||||
scikit-learn
|
||||
pydotplus
|
||||
matplotlib
|
||||
matplotlib
|
||||
scipy
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
csvdiff.py file1.csv file2.csv
|
||||
Streams both files; prints the first differing line or
|
||||
‘No differences found’. Uses O(1) memory.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from itertools import zip_longest
|
||||
from pathlib import Path
|
||||
|
||||
def open_checked(p: str):
|
||||
print(p)
|
||||
path = Path(p)
|
||||
try:
|
||||
return path.open("r", newline=""), path
|
||||
except FileNotFoundError:
|
||||
sys.exit(f"Error: {path} not found")
|
||||
|
||||
def human(n: int) -> str:
|
||||
return f"{n:,}"
|
||||
|
||||
def main(a_path: str, b_path: str) -> None:
|
||||
fa, a = open_checked(a_path)
|
||||
fb, b = open_checked(b_path)
|
||||
|
||||
with fa, fb:
|
||||
for idx, (ra, rb) in enumerate(zip_longest(fa, fb), 1):
|
||||
if ra != rb:
|
||||
print(f"Files differ at line {human(idx)}")
|
||||
if ra is None:
|
||||
print(f"{a} ended early")
|
||||
elif rb is None:
|
||||
print(f"{b} ended early")
|
||||
else:
|
||||
print(f"{a}: {ra.rstrip()}")
|
||||
print(f"{b}: {rb.rstrip()}")
|
||||
return
|
||||
print("No differences found")
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
sys.exit("Usage: csvdiff.py file1.csv file2.csv")
|
||||
main(sys.argv[1], sys.argv[2])
|
||||
@@ -0,0 +1,600 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6741d9c2",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# IoT Traffic Diversity Notebook\n",
|
||||
"\n",
|
||||
"Point `DATA_DIR` at the folder that contains your per‑day CSVs.\n",
|
||||
"The notebook will\n",
|
||||
"\n",
|
||||
"1. Load every CSV (optionally recursive)\n",
|
||||
"2. Build cumulative statistics as each file is added\n",
|
||||
"3. Show how diversity (unique `(src,dst)` pairs, classifications, protocols)\n",
|
||||
" grows with more days of traffic\n",
|
||||
"4. Plot helpful bar/line charts\n",
|
||||
"\n",
|
||||
"Run the cells in order or execute the whole notebook."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2ce04e2d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pathlib import Path\n",
|
||||
"import pandas as pd\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from itertools import accumulate\n",
|
||||
"\n",
|
||||
"DATA_DIR = Path(\"../data/processed\") # <- change if your location is different\n",
|
||||
"RECURSIVE = True # set False if flat directory"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "afc490a3",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### 1 Collect all CSV files"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "13a6b6ad",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Found 9 CSV files\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"pattern = \"**/*.csv\" if RECURSIVE else \"*.csv\"\n",
|
||||
"csv_files = sorted(DATA_DIR.glob(pattern))\n",
|
||||
"print(f\"Found {len(csv_files)} CSV files\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c0bcbcc7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### 2 Load each file and compute per‑file stats"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "774b8037",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>file</th>\n",
|
||||
" <th>packets</th>\n",
|
||||
" <th>unique_pairs</th>\n",
|
||||
" <th>unique_class</th>\n",
|
||||
" <th>unique_proto</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>16-09-23\\16-09-23.csv</td>\n",
|
||||
" <td>947072</td>\n",
|
||||
" <td>27824</td>\n",
|
||||
" <td>18</td>\n",
|
||||
" <td>7</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>16-09-24\\16-09-24.csv</td>\n",
|
||||
" <td>799235</td>\n",
|
||||
" <td>18587</td>\n",
|
||||
" <td>17</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>16-09-25\\16-09-25.csv</td>\n",
|
||||
" <td>537650</td>\n",
|
||||
" <td>16359</td>\n",
|
||||
" <td>17</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>16-09-26\\16-09-26.csv</td>\n",
|
||||
" <td>573848</td>\n",
|
||||
" <td>19850</td>\n",
|
||||
" <td>17</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>16-09-27\\16-09-27.csv</td>\n",
|
||||
" <td>527035</td>\n",
|
||||
" <td>14889</td>\n",
|
||||
" <td>17</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>5</th>\n",
|
||||
" <td>16-09-28\\16-09-28.csv</td>\n",
|
||||
" <td>2019000</td>\n",
|
||||
" <td>180877</td>\n",
|
||||
" <td>21</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>6</th>\n",
|
||||
" <td>16-09-29\\16-09-29.csv</td>\n",
|
||||
" <td>738906</td>\n",
|
||||
" <td>29700</td>\n",
|
||||
" <td>20</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>7</th>\n",
|
||||
" <td>16-09-30\\16-09-30.csv</td>\n",
|
||||
" <td>802226</td>\n",
|
||||
" <td>25366</td>\n",
|
||||
" <td>20</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>8</th>\n",
|
||||
" <td>16-10-01\\16-10-01.csv</td>\n",
|
||||
" <td>736136</td>\n",
|
||||
" <td>30733</td>\n",
|
||||
" <td>20</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" file packets unique_pairs unique_class unique_proto\n",
|
||||
"0 16-09-23\\16-09-23.csv 947072 27824 18 7\n",
|
||||
"1 16-09-24\\16-09-24.csv 799235 18587 17 5\n",
|
||||
"2 16-09-25\\16-09-25.csv 537650 16359 17 5\n",
|
||||
"3 16-09-26\\16-09-26.csv 573848 19850 17 5\n",
|
||||
"4 16-09-27\\16-09-27.csv 527035 14889 17 5\n",
|
||||
"5 16-09-28\\16-09-28.csv 2019000 180877 21 6\n",
|
||||
"6 16-09-29\\16-09-29.csv 738906 29700 20 6\n",
|
||||
"7 16-09-30\\16-09-30.csv 802226 25366 20 5\n",
|
||||
"8 16-10-01\\16-10-01.csv 736136 30733 20 5"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"per_file_stats = []\n",
|
||||
"for f in csv_files:\n",
|
||||
" df = pd.read_csv(f)\n",
|
||||
" stats = {\n",
|
||||
" \"file\": f.relative_to(DATA_DIR),\n",
|
||||
" \"packets\": len(df),\n",
|
||||
" \"unique_pairs\": df[[\"src\", \"dst\"]].drop_duplicates().shape[0],\n",
|
||||
" \"unique_class\": df[\"classfication\"].nunique(),\n",
|
||||
" \"unique_proto\": df[\"protocl\"].nunique(),\n",
|
||||
" }\n",
|
||||
" per_file_stats.append(stats)\n",
|
||||
"\n",
|
||||
"stats_df = pd.DataFrame(per_file_stats)\n",
|
||||
"stats_df"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6e2f635d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### 3 Cumulative diversity growth"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "534244fe",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>file</th>\n",
|
||||
" <th>packets</th>\n",
|
||||
" <th>unique_pairs</th>\n",
|
||||
" <th>unique_class</th>\n",
|
||||
" <th>unique_proto</th>\n",
|
||||
" <th>cum_packets</th>\n",
|
||||
" <th>cum_pairs</th>\n",
|
||||
" <th>cum_class</th>\n",
|
||||
" <th>cum_proto</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>16-09-23\\16-09-23.csv</td>\n",
|
||||
" <td>947072</td>\n",
|
||||
" <td>27824</td>\n",
|
||||
" <td>18</td>\n",
|
||||
" <td>7</td>\n",
|
||||
" <td>947072</td>\n",
|
||||
" <td>27824</td>\n",
|
||||
" <td>18</td>\n",
|
||||
" <td>7</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>16-09-24\\16-09-24.csv</td>\n",
|
||||
" <td>799235</td>\n",
|
||||
" <td>18587</td>\n",
|
||||
" <td>17</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" <td>1746307</td>\n",
|
||||
" <td>46411</td>\n",
|
||||
" <td>35</td>\n",
|
||||
" <td>12</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>16-09-25\\16-09-25.csv</td>\n",
|
||||
" <td>537650</td>\n",
|
||||
" <td>16359</td>\n",
|
||||
" <td>17</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" <td>2283957</td>\n",
|
||||
" <td>62770</td>\n",
|
||||
" <td>52</td>\n",
|
||||
" <td>17</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>16-09-26\\16-09-26.csv</td>\n",
|
||||
" <td>573848</td>\n",
|
||||
" <td>19850</td>\n",
|
||||
" <td>17</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" <td>2857805</td>\n",
|
||||
" <td>82620</td>\n",
|
||||
" <td>69</td>\n",
|
||||
" <td>22</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>16-09-27\\16-09-27.csv</td>\n",
|
||||
" <td>527035</td>\n",
|
||||
" <td>14889</td>\n",
|
||||
" <td>17</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" <td>3384840</td>\n",
|
||||
" <td>97509</td>\n",
|
||||
" <td>86</td>\n",
|
||||
" <td>27</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>5</th>\n",
|
||||
" <td>16-09-28\\16-09-28.csv</td>\n",
|
||||
" <td>2019000</td>\n",
|
||||
" <td>180877</td>\n",
|
||||
" <td>21</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" <td>5403840</td>\n",
|
||||
" <td>278386</td>\n",
|
||||
" <td>107</td>\n",
|
||||
" <td>33</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>6</th>\n",
|
||||
" <td>16-09-29\\16-09-29.csv</td>\n",
|
||||
" <td>738906</td>\n",
|
||||
" <td>29700</td>\n",
|
||||
" <td>20</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" <td>6142746</td>\n",
|
||||
" <td>308086</td>\n",
|
||||
" <td>127</td>\n",
|
||||
" <td>39</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>7</th>\n",
|
||||
" <td>16-09-30\\16-09-30.csv</td>\n",
|
||||
" <td>802226</td>\n",
|
||||
" <td>25366</td>\n",
|
||||
" <td>20</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" <td>6944972</td>\n",
|
||||
" <td>333452</td>\n",
|
||||
" <td>147</td>\n",
|
||||
" <td>44</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>8</th>\n",
|
||||
" <td>16-10-01\\16-10-01.csv</td>\n",
|
||||
" <td>736136</td>\n",
|
||||
" <td>30733</td>\n",
|
||||
" <td>20</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" <td>7681108</td>\n",
|
||||
" <td>364185</td>\n",
|
||||
" <td>167</td>\n",
|
||||
" <td>49</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" file packets unique_pairs unique_class unique_proto \\\n",
|
||||
"0 16-09-23\\16-09-23.csv 947072 27824 18 7 \n",
|
||||
"1 16-09-24\\16-09-24.csv 799235 18587 17 5 \n",
|
||||
"2 16-09-25\\16-09-25.csv 537650 16359 17 5 \n",
|
||||
"3 16-09-26\\16-09-26.csv 573848 19850 17 5 \n",
|
||||
"4 16-09-27\\16-09-27.csv 527035 14889 17 5 \n",
|
||||
"5 16-09-28\\16-09-28.csv 2019000 180877 21 6 \n",
|
||||
"6 16-09-29\\16-09-29.csv 738906 29700 20 6 \n",
|
||||
"7 16-09-30\\16-09-30.csv 802226 25366 20 5 \n",
|
||||
"8 16-10-01\\16-10-01.csv 736136 30733 20 5 \n",
|
||||
"\n",
|
||||
" cum_packets cum_pairs cum_class cum_proto \n",
|
||||
"0 947072 27824 18 7 \n",
|
||||
"1 1746307 46411 35 12 \n",
|
||||
"2 2283957 62770 52 17 \n",
|
||||
"3 2857805 82620 69 22 \n",
|
||||
"4 3384840 97509 86 27 \n",
|
||||
"5 5403840 278386 107 33 \n",
|
||||
"6 6142746 308086 127 39 \n",
|
||||
"7 6944972 333452 147 44 \n",
|
||||
"8 7681108 364185 167 49 "
|
||||
]
|
||||
},
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"stats_df[\"cum_packets\"] = list(accumulate(stats_df[\"packets\"]))\n",
|
||||
"stats_df[\"cum_pairs\"] = list(accumulate(stats_df[\"unique_pairs\"]))\n",
|
||||
"stats_df[\"cum_class\"] = list(accumulate(stats_df[\"unique_class\"]))\n",
|
||||
"stats_df[\"cum_proto\"] = list(accumulate(stats_df[\"unique_proto\"]))\n",
|
||||
"stats_df"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c3022457",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### 4 Plots"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "53dec127",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAlUAAAHHCAYAAACWQK1nAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAb1VJREFUeJzt3Qd4k9X3B/DTvSfdhbZAoaVl7w0KtCLIVBERkKEyFZniT0X8qyC4UBGc4AAFFBSZZc+yd4FCESijpVC66B7v/zmnJCZtgRbSJk2/n+cJTd73zZubUXJ677nnmiiKohAAAAAAPBLTR7s7AAAAADAEVQAAAAA6gKAKAAAAQAcQVAEAAADoAIIqAAAAAB1AUAUAAACgAwiqAAAAAHQAQRUAAACADiCoAgAAANABBFUAevTuu++SiYkJGapLly5J+xYvXqzvphi0zp07U/369amyvncvvvgiBQQEUFXGz59fB4BHgaAKQEf4y4u/xFQXa2tr8vHxofDwcPriiy8oLS2NjMG6deskGKxqrl+/Ls/72LFj+m4KABgoBFUAOvbee+/RL7/8QgsWLKDx48fLtgkTJlCDBg3oxIkTWse+9dZblJmZSYbK399f2jd48GCtoGrmzJlUFYMqft4IqoxTdHQ0fffdd/puBlRy5vpuAICx6d69OzVv3lx9e/r06bR161bq2bMn9erVi86cOUM2Njayz9zcXC4ViddQz8rKUrfhflQ9boYuPT2d7Ozs9N0Mo5aXl0cFBQVkaWlJxsjKyuqBx+BzBg+CniqACvD444/T22+/TZcvX6Zff/31njlVnJfz2GOPFbs/f5n5+vrS008/rbXt888/p9DQUAl8PD096ZVXXqGkpKRiuSIc0G3cuFGCPQ6mvvnmG9m3adMmat++PTk7O5O9vT0FBQXRm2++ec+8HM45mT9/vlzXHOrkQI0fp3fv3sXazgGck5OTtO1+uEfs1VdfJTc3N3JwcJAA9Nq1a3J+zeFG1Wt2+vRpev7558nFxUWeg+qL///+7/+odu3a8iXJbeLnk52drb7/xIkTqVq1atJmFe5R5HPyMK3KjRs3ZBv3OG7fvp1atGgh24cNG6Z+3kXzlbhN/P7Z2trK+zVnzpxizzM2NpbOnj1LD5KTk0PvvPMONWvWTF4//jLv0KEDbdu2rdixycnJ8t7wcfxeDh06VLaV5K+//pLPGX9m+OeqVauKHaN63z/++GP5jKleT35+jNvPn0VXV1c5D3+uVq9erXWO3Nxc6dmrU6eOHMOvOb9P/JlTiY+Pl9ezevXqcn5vb2/5DPHj3w8/V/68/vvvvzK8zq8ND7VzL7Hm+8r4ObRt21Yenz/7/Hr+8ccfD8ypUg3n79ixg8aMGUMeHh7STsZD+dz7zPfhdvO+bt260ZEjR+7bbjB+6KkCqCA8hMZf8BEREfTSSy+VeMyAAQMkaOAvGy8vL/X23bt3y/DTc889p97GQQr/x89fShyMXLx4kb766is6evQo7dmzhywsLLSGNgYOHCj34cfm4CkqKkqCrYYNG8qXEX85xMTEyH3vhe/P7eAvRh7iVOEvnxdeeEGCiNu3b8uXrco///xDqampsv9++Att+fLl8jq1bt1avsx69Ohxz+OfeeYZ+cL+8MMP1V+kI0eOpJ9++km+8CdNmkT79++nWbNmSe+gKnjgwOSzzz6T569KLt+1axeZmprKT34tVdtYx44dJdDj14iDnJdfflnOwfjLWoWD2SeeeIL69etHzz77rHxxT5s2TYZ9ufdSZciQIfLcin75F8Wv2ffffy/vG79n/EX+ww8/SBBx4MABaty4sRzH5+FAhD8jo0aNonr16slz5cCqKP7s9e/fn0JCQuR1SUxMVAc1JVm0aJEExfyc+fPB7yu/bu3atZOg8Y033pCAht+3Pn360J9//kl9+/aV+/LnmB+D35OWLVvK8zl06JAEHhyAMG4Ln4+DWg5QEhIS5LPFgeeDEufz8/Pl9ebPCn/uNmzYQDNmzJDAmt8rlXnz5kmAPmjQIAlUf//9d/nsrFmz5r6fLxUOqNzd3eW9554qxq8zv7/jxo2T15JfR379+XPWtGnTB54TjJgCADqxaNEi/pZUDh48eM9jnJyclCZNmqhvz5gxQ+6jEh0dLbe//PJLrfuNGTNGsbe3VzIyMuT2rl275LglS5ZoHbdhw4Zi2/39/WUb79P02WefyfabN2/es70XL16UY/i5qYwdO1arzUXbvmDBAq3tvXr1UgICApSCgoJ7Ps7hw4flvhMmTNDa/uKLL8p2fp2KvmYDBw7UOvbYsWOyfeTIkVrbJ0+eLNu3bt0qtxMSEuT2119/LbeTk5MVU1NT5ZlnnlE8PT3V93v11VcVV1dXdbv5fS36Wqh06tRJ9v3888/qbdnZ2YqXl5fSv3//Eo99kLy8PDmHpqSkJGnj8OHD1dv++usvOd+cOXO07tuhQ4di7W3cuLHi7e0tz1klIiJCjuPPSdH33dHRUV4vTV26dFEaNGigZGVlqbfxa9S2bVulTp066m2NGjVSevTocc/nx8+FH2Pu3LlKWQ0dOlTuO378eK028ONZWlpqfaZVvzMqOTk5Sv369ZXHH39cazs/fz5v0d/n9u3by+tZ9PeYfw8AisLwH0AF4iGL+80CrFu3rvRALFu2TOsvcv6r+KmnnlLnQa1YsUKGevgv/lu3bqkvPLTBj1F0iKhmzZrSw6GJh4nY33//LUOJj4rb3qpVK1qyZIl6G/darV+/XnoJ7lc6gnsZVL0CmlSJ/iXh3gJNnECvGt7TxD1WbO3atfKTex2Cg4Np586dcpt75szMzGjKlCky5Hf+/Hl1TxUPV5W25AW/7pq9cZx7xD00PESliYcSH9RLxbhNqvwlfn/4teReGB5q0xxm4ufNeXmjR4/Wum/R1y4uLk6S7LkHiz87KvwZ4t6WknBPEr9eKtwGzg/knjj+HKs+d9xTw58vfu14yFb1+eJeKNXrWRR/lvn58etRdMi6tLinSIXfJ77NvVGbN2/WehwVfpyUlBTpaSztUB33EvLrqYmfG/eCcq8tgCYEVQAV6M6dO5IvdD88BMhf9KovJ/7S4WER3q7CX1T85cC5HPylp3nhx+DjiwZVJT0OD+Pw8AznY/HQIg/jPEqAxUNb3HbOHVMFf5xbozl7sCR8PA+/FW1nYGDgPe9T9FjVOYreh4dR+UtQ1SbGX6qq4T3+yYEKX3h4i2/zUNXx48fVw3ylwUNoRQMwzvd62ICB8VAmD8+qcpL4/eXgkN97zefNuUgc1GniIV5NqufPQ6ZFFT32Xq8xDw9zQMj5gUU/dzz0xlSfPR6C47wuDrZ5CJSDVs3Zrzyc+NFHH0nQzZ8/HmblYTwe+i4Nfq9r1aqltY0fi2nmZPEwHw8R8mvI7y+3lfPkNF/D+ynpd4fbeerUKapRo4YEzjzUWTR4hqoJQRVABbl69ar8R36/QEEV7PAXFwckjAMd7lng/BEVDnw4oOL8k5IumjklrKSZfryNe2v4r3oOevgLjx+bey64d+xhcGDGuVyq3ipOyudg5V5f2o/iXrMXS9OzxD1QHLTyFyEHURw88f14O9/eu3evvMZlCaqK9maolKZXqiT82nGeGSeJcy4V9+bxe8uTHnTRs/gwr7HqcSdPnnzPz57q881B0oULF+jHH3+U3DXOD+N8I/6pwsne586dk9wrDno4WOOcMM4L1AV+Lzmfis/99ddfS68et5EnOJT2fSnpc8Y9dfzZ+fLLLyVBfu7cuTJhhANEqNoQVAFUEFVid9FhuJL+Mua/fnkIkId7Vq5cKUnAmlO++YuWh1y4p6lr167FLo0aNSr1X/tdunShTz/9VGZ2ffDBBzK8U9IMs9IELdwTwMm/HFRxzwj3Wj2ol0pVD4u/sDnZvmjPSGmpzlF0uImH9LjHhPerqIIl/oI9ePCg+jYHAvxFzBdOwObh1NI87/LAQ77cE8PvP7+G/Lnh95YTxzXx8+KhPe6h1MSTE4oex0oajit67L2oeoY4cC7pc8cXzZ5Y/jxwIvxvv/1GV65ckV63ooVj+bPMQ7ScRM+9Pzx898knnzywLfxeF+0d4gCNqZLcOXGeAyqe+Tp8+HCZMMBt1AXuHeThap5NyZ9b7knk3x+o2hBUAVQADlR4qj8HTJxf9CDcY7Rv3z75K59zVjSH/lR/KXNvEp+zKA7E7jWdXhPnxxSlmlGmWYKgKFWdnns9BgcAHKDxcA/33mjOWLwXVaDJvQmauCegtJ588kn5ySUANHHAyDRnevH7wLPXeBYgD09ycMo4uOLeFQ5oeMhIs4bYg553aZW2pIKq50uzR4XzeCIjI4s9b37PeUhLhT8bRV87DgL4/eUhRc2hLw4sVaUSHoR7R3lJHi7JwYFcUTdv3lRf56BfEw9Pci+W6rOVkZFRLEDkAIuDsvt9/jTxbFcVfp34Ngd8/IeC6jXkYFiz55WHBjkQelh8rqJDh/y6cI9VadsNxgslFQB0jIcA+EuTv+i4l4QDKv7i4p4CruVTmmKaHDTxEAtf+K/9on9dd+rUScob8LAJJx+HhYXJlwn3QvCwIU8j16xpVRIeIuThPw42uG2cC8NBDecGqeo+lUTVe8OlBzgYKho48fn4r3ZuB/cM8BfOg/A5OSmaAyL+MlaVVFD1PJSml4h75zgJ+9tvv5XAh18jLj3AQQT39BWt/8UBFE+v53wfzn1iPDzFwRM/Lg8RFf3C59yshQsXyhc/H8eJ+SXl3NxPaUsqcLkL7qXiEgX8mnJvCD82J5Vr9krxBAYOCrm8AQcMvJ/vV1LOEH9e+Fz8/nLPDQfWHHzx0FXRnq574TplfH9+3TiJm3uv+HPOwR4PcXMuGuN2cADG7y1/hrmcgqoMAePXmIMf/qzzsRzAcikIPldpAnH+PeIhUX7P+X3g3zvON+OyJarken6uHFTz0Dm/n/wZ5/ZzcFd0dYPS4gR9/h3h3y/+zHGwyEPLine truncated
|
||||
"text/plain": [
|
||||
"<Figure size 640x480 with 1 Axes>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"plt.figure()\n",
|
||||
"plt.plot(stats_df.index + 1, stats_df[\"cum_pairs\"])\n",
|
||||
"plt.xlabel(\"Files processed\")\n",
|
||||
"plt.ylabel(\"Cumulative unique (src,dst) pairs\")\n",
|
||||
"plt.title(\"Diversity growth: address pairs\")\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "49a6b2be",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAHHCAYAAACRAnNyAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAXrtJREFUeJzt3Qd8U1X7B/CndBdKy6bQslpWgTIF2XuLMhRReAX1RQEZgqDgYKgsURFfZagMFRVlyVC27L33Xi1QKFBoC6U7/8/v1OSf7hbS3pvk9/18As3NbXJyk+Y+ec5zznEwGAwGISIiItKhfFo3gIiIiCgjDFSIiIhItxioEBERkW4xUCEiIiLdYqBCREREusVAhYiIiHSLgQoRERHpFgMVIiIi0i0GKkRERKRbDFTIKpQrV06eeeYZsQX9+vVTz4csy8HBQcaPH691M+hfeI/jvW5J/NuxTwxUKMcWLFigTgrGi5ubm1SqVEkGDx4st27dEr3btWuXOqHdv39f66YQUSo3btxQf59HjhzRuimkE05aN4Cs18cffyzly5eXmJgY2bFjh8yaNUv+/vtvOXHihHh4eIieA5UJEyaob2fe3t5aN4eIUgUq+PtE5qRWrVopbvv+++8lKSlJs7aRNhio0GPr2LGj1KtXT/383//+V4oUKSJffvmlrFixQl566SWtm0dWKiEhQZ2MXFxctG4K6Yyzs7PWTSANsOuHLKZVq1bq/8uXL6v/P//8c2nUqJEKYNzd3aVu3bqyZMmSdH934cKFUr9+fZWJKVSokDRr1kzWr1+f6eP9+OOP4uTkJKNGjTJt27t3r3To0EG8vLzUfTVv3lx27txpuh0pZeP+yAYZu6+uXLmS7mOgO6tAgQISHR2d5jYEYyVLlpTExETTtpkzZ0q1atXE1dVVSpUqJW+99VaWXUxbtmxRbcD/5tAmbEdXmxGyQGhPcHCwqtnBz6VLl5Zvv/1W3X78+HH1OuTPn1/Kli0rv/76a5rHQ3vefvtt8fPzU+0MCAiQqVOnZuubqrFWCK8Nvu2i2y8wMFCWLVv2WI9jfI54r3z11Vfi7++v9j116lSGbYiNjZXhw4dLsWLFxNPTU5599lm5du1amv2uXr0qgwYNksqVK6v3H96HL7zwQorX+tKlS+rxp0+fnm7mDbf99ttv6bYD3Zx4/+Hbf2pnz55Vv/vNN9+o6/Hx8Wq/ihUrqmOGtjRp0kQ2bNiQ4fM0P454vjj2ODa+vr7yyiuvyJ07d1J0xaZ+D6f3vmrRooVUr15djh07pv428DeC18X4d7l161Zp0KCBOl44bhs3bsxWjQj+rvBYmQkPD5eRI0dKjRo11Pu2YMGC6svO0aNHU7T5qaeeUj+/+uqrpr9P49+A+ePjmBYuXFjtl1pkZKQ6zng88/fNuHHj1PPFccT78t1331XbSd8YqJDFXLx4Uf2PD2GYMWOG1K5dW3URTZo0SX2o40Tx119/pfg9fID/5z//Ud+WsC+u40Pkn3/+yfCxvvvuO/UBNXr0aJk2bZrahv0R4OBDCh9IeEx8yOPEvW/fPrVP9+7dTdkenJx+/vlndcFJLz0vvviiPHz4ME2bEbisWrVKnn/+eXF0dDR9WCMwQYDyxRdfSI8ePWTOnDnSrl079aFqKQiM8AGPY/TZZ5+pD24EVPgwR5CGLBcCApzEcUIzBo7GduMEhcAQt3399dfSuHFjGTNmjIwYMSJbj3/+/Hl1XNCGyZMnm15X85NuTh9n/vz58r///U/eeOMNdexwAsoIsncIanBcp0yZot43nTt3TrPf/v37VbDRq1cv9fgDBgyQTZs2qZO1MfCsUKGCatcvv/yS5vexDcfwueeeS7cdJUqUUM/xjz/+SHPb77//rt4XOC7G9wbe1y1btlTBywcffCBlypSRQ4cOSWYePHggTZs2VccGzxd/U3geZ86cSTc4y4579+6pYBMBCd4/OGnjGKHN+L9Tp07quOJ9j/d3VFSUWAKCwj///FM9NjKv+MKAwBrHEN09ULVqVfUZAHgvGP8+8XedGl73bt26qfuMi4tLcRu2IQDB8wEExwhoERB36dJFHc+uXbuqzwC8l0nnDEQ5NH/+fAPeOhs3bjTcvn3bEBISYli0aJGhSJEiBnd3d8O1a9fUftHR0Sl+Ly4uzlC9enVDq1atTNvOnz9vyJcvn6Fbt26GxMTEFPsnJSWZfi5btqyhc+fO6ucZM2YYHBwcDJ988kmKfStWrGho3759it9DG8qXL29o27atadu0adNU+y9fvpzlc8V9lS5d2tCjR48U2//44w91H9u2bVPXw8LCDC4uLoZ27dqleB7ffPON2m/evHmmbX379lXPx2jz5s1qH/xvDu3Ddhxv89/FtkmTJpm23bt3Tx13HBO8DkZnzpxR+44bN860Dccsf/78hnPnzqV4rNGjRxscHR0NwcHBmR4PtBv3uXTpUtO2iIgIg4+Pj6F27do5fhzjcyxYsKA6hlk5cuSI2n/QoEEptr/88stpnmvq9x/s3r1b7ffTTz+Zts2ZM0dtO336dIr3atGiRdXxzozxd48fP55ie2BgYIr3ec2aNU3v35wYO3asuv9ly5aluc34Pjf+PaZ+P6f3vmrevLna9uuvv6Z5n+DvcM+ePabt69atS/f9Z/7eNcJxT306wX7mxy8mJibN3zja7Orqavj4449N2/bv35/mcTN6fGMbV61alWK/Tp06GSpUqGC6/vPPP6vnt3379hT7zZ49W/3+zp070zwW6QczKvTY2rRpozIR+GaPby5I5y5fvlx1RQDSx+bf4iIiItS3Q/Nvkfjmg287Y8eOlXz5Ur4d00sl4xvgsGHDVMbgww8/NG3HCAF803/55Zfl7t27Ki2OC74Vtm7dWrZt2/ZYRXhoA74Vo0gY326N8O0TzxPpe0CKHN/q0NVh/jz69++vUtypMzJPClkFIxQEI02P7p6ePXuatmMbbsM3WaPFixer1wDda8ZjhAteS2RqcJyygowRvska4fkha3L48GG5efPmYz0Osk8ZZbXM4XWAoUOHptiO456a+fsPGS28L5D2xzExfw/imKGbwDyrsm7dOtXePn36ZNoeZOiQUcL7wQjF5Oi6Mv+mjsc8efKkeo/mxNKlS6VmzZopjrdRVl0tGcHfqTHTYP4+QTYDWRYj48/m758ngcyN8W8D7wG8HmgLHj+rzFJGkC0tWrRoiuOPzxpk98yPP96PeH5VqlRJ8X40dldv3rz5iZ8f5R6bCVTwwYeUHj5E8QeME2BOGQwGlRrEUFv8UeFENHHixFxpry1AXQQ+EPBHjg9mfKC1b9/edPvq1avl6aefVicBpPJxIsLIIAQs5t1F+PBCnUNW0H/+3nvvqYt5XQoYTwB9+/ZVj2N++eGHH1Qa2PxxcwIfeI8ePZKVK1eq6whYcMJEAGM8WaAeAvChaw4FoeheMN5uCTieqU/qqMlB7ULqkxe244Pb/DitXbs2zTFCAAFhYWFZPj5O9qkfB38zYKyTyOnjoF4oO3Ac8X5BLYu51Mcd8JohADbWyOCEhjagO9D8vYCTND47zOt5ELTg7994IssI7hOBsHn3D06aCF4QxBihOwOPi+OEGg28f1EnkhX8faCmxJIyep/gOKXeBubvnyeBLwroakGdjvnrgePwuH+bOM4IclHAb6w1Qb0UAlPzQAXvRwSKqd+Pxvdtdt73pB2bGfWDb8745vHaa6+l+IDICXxTR5EgghV8mKD4CxdKH4pfjaN+Utu+fbvqE0bfMgpMfXx8VJ8yahHSK/DMDhSp4sMefdZvvvlmipObMVuCepXUQxqN8O3tcSDYQh0ITkbI2KA2BSdBS/VtZ/TN2LxI15yxJia72xGAmx+ntm3bqiLC9Bg/uJ9UTh/HPPthKUOGDFHvN2RbGjZsqE68ONbIJqTOriEjhG/dqGnB3z6CUhTips7ypQf3h3opZPXw3sP7BMELTsRG+DtA0IETKj5jEDzjpD179uwU2TG9v39y+ljmUDP20Ucfqc/oTz75RH15wfHF6/MkQ45x/FELtmbNGlV3guOPzAnOB0a4f7yuqI1JT+ogjfTFZgIVFPbhkhFE2yhgQwU/Tnb4loLuAxTWwenTp9W3faRtjd/Osvstj9JPWeObP1Lo+PZkhBOHOXwzxocIMjIZBRhG+ODH6AR0t+BEgLlbkEEz3o+xG8L4rT0jj5MyR/cAChlRqItvzAhcEMAYYYSNcbQHMihG6A5CMWtmbUL3CKQeHWTJLIwRjhMyQlkdo8xcuHBBnbzMj+O5c+fU/8YRGZZ4nPTgOOP9gpO+eRYFxz01vFeQYUNxrhHm/ElvFBaKkPENG5kUdHmg2BYF3tmBkyMCZ2P3A44FioZTM45QwQXHBsELimwzC1RwHPGZlJm8fP/gsdI7ftl5LLweKCaeO3duiu24P/OgLqd/nziO+CKE44/PBhTV47M+9XHE6CJ8bjxulxlpx2a6frKCURG7d++WRYsWqVQj0vb4cDJ2GeBbMk4w6K5AgIIPXHyAMKPyePDtDB8I5t+00C2QuksOH/L4VoXUeOpvVebf5MzT1qgHQUYD39jRzw0Y+owPI2TDzGtJjG7fvm36GbUckJOZaZE9QbCLIdHo0jCvBQGckNHNg9El5u3GhzLS2umNSjE/+eJ4pa7bQCbK0tBu/B0ggEwNxwNzmGQFIzRQi2SE4O2nn35SgSaGa1vqcdJj/DKC42wOo4BSwzFN/R7CaI/0vv2jCwGjwfBtHKOn8O07KCgoW21C1xG6PPG7+HzB+wDva3PG96l5dg9daFkNjUW3Bk6w5sfbyPjcjEG6+fsHzxEj4ywNj4X3s3m3VWhoaLrty87rgSzW9evXU2zL6d8nPj8wOgmf4ci24r2VOtuJ9yMeBxPGpYbPEmTkSb9sJqOSGcw5gW/y+N/4DRzj63HCwXakJFFfgW8F+MPBhy7+0DF3Af4AMhsmS+nDiRlpVgSD6C5BHzBLine truncated
|
||||
"text/plain": [
|
||||
"<Figure size 640x480 with 1 Axes>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"plt.figure()\n",
|
||||
"plt.bar(stats_df.index + 1, stats_df[\"packets\"], label=\"packets/day\")\n",
|
||||
"plt.plot(stats_df.index + 1, stats_df[\"cum_packets\"], marker=\"o\", label=\"cumulative\")\n",
|
||||
"plt.xlabel(\"Files processed\")\n",
|
||||
"plt.ylabel(\"Packets\")\n",
|
||||
"plt.title(\"Packet volume per day vs cumulative\")\n",
|
||||
"plt.legend()\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "851f4bd1",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Loaded 7,681,108 packets from 9 files\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<matplotlib.legend.Legend at 0x14ec67d9a90>"
|
||||
]
|
||||
},
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAABOQAAASdCAYAAADg7IMVAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzs3Ql8XGW9P/7PObPPZE+btGm6QmmBln1fZbHIrV5RUFA2C7jwR72AiuLlV3C5oiICXhBcLgW3C3IRFJBNKIuy76XQ0pZClzT7Pvucc/6v75Oc6WSapFnOZCaTz1vDk2fm5MyZSfqcOd/5Pt9HsyzLAhEREREREREREU0IfWIehoiIiIiIiIiIiAQDckRERERERERERBOIATkiIiIiIiIiIqIJxIAcERERERERERHRBGJAjoiIiIiIiIiIaAIxIEdERERERERERDSBGJAjIiIiIiIiIiKaQAzIERERERERERERTSAG5IiIiIiIiIiIiCYQA3JEREREeXTddddhwYIFcLlcOOCAA9RtqVQKV1xxBWbPng1d13Haaaep2zVNwzXXXJPnIyYiIiKi8WJAjoiIqEDccccdKuBif/n9fuy111746le/iqamJkcf60c/+hHuv//+EW//97//PWeBoI985CMDnndVVRUOPfRQ3H777TBN07HHeeedd9Rz+OCDD4bdTu7PPJ7hvna3r9157LHHVODt6KOPxqpVq9TvRchzl0DdGWecgTvvvBOXXXYZilV3dze+973vYf/990dJSQkCgQCWLFmCb3/722hoaBiw7QMPPIDjjz8eNTU1CAaDKpD52c9+Fo888oi6/+c//7n6vfzjH/8Y8vF+85vfqG3+9re/5fy5EREREQ1FsyzLGvJeIiIimtCA3IoVK/D9738f8+fPRywWwz//+U/8/ve/x9y5c/H222+rIIQTJPAhwR55zJGQoOAtt9yCXLxtkIDcpk2bcO2116p+S0sLfve73+GNN95QQZkf//jHjjzO//3f/+Ezn/kMVq9erR5zKOFwGPfdd9+A266//nps27YNN9xww4DbP/WpTyEUCo35mL7zne+owFs0GoXX603fftZZZ6nfvTxmJvmbcLvd6qsYvP/++zj55JOxZcsW9bs55phj1Ovw1ltv4X//939VcPa9995T2/7sZz/Dt771LRWQ++QnP6n+LWzcuFEF3ySYJ3/LEsCTrMLzzz9fBTUHc8IJJ2DNmjXYsWMHPB7PBD9jIiIioj7F8W6OiIioiJx66qk45JBD1PcXXXQRqqurVebPX//6V3zuc58b834lmCYBHclAKjTl5eU455xz0v0vf/nLWLRoEW6++Wb84Ac/GFfgRJ5zZrBrdyTAlnks4q677kJHR8cut4/39W1ublbbZx+f3F5RUbHL9pI1OZlIcHOogKVMy/30pz+tsj+feuopFYzL9F//9V/4yU9+kt5W/g4++tGPqqzCbPJ6ibq6OhVw+8tf/oJbb70VPp9vwHbbt2/HM888gy996UsMxhEREVFeccoqERFRgTvxxBNVu3nz5gHBiT322EMFHObNm4fvfve7iMfjA35Obv/4xz+ORx99VAX4JPDzq1/9Sk3Xk0CJTIW0p15+4QtfGPLx5T7JjhOZ0zVtsq9vfOMbKjNJjkcCaZLNNJ5sOsl+OuKII9S+JWPOzqaSLCrJmrLvf+ihhwb8nAR25NgkgHbVVVdh1qxZattf/OIX6meFBGzs5yDbj9VQr6+Q6afye5OplfKa7LPPPipAlEkeX7aT52gfjz1tWbL41q5du8txDlZDToJMF154oQpGyWNJduXFF1+MRCKx22m58nuSrD/JwJTjl+wzycTMtm7dOpVRKa+9BAXl+WZP+bSP/emnn8b/9//9f+q519fXD3kM9957L958803853/+5y7BOFFWVqaCcqK1tVVNbZWpvYORx7JJ0LSrq2uXvw0hfxcyDfrss88e8riIiIiIJgIz5IiIiAqcTOcUkilnZ81JME0CJBIIe/HFF9V0z3fffXeXqZbr169XWXWScfbFL35RBctkCqzs47DDDlOZQkKCe0ORn5WpgI8//rj62UwSdPv3f/93FUCSoJAsSiABKplaKIGi7CmeoyEBOFnoQDLFJIvqqKOOQiQSwde//nX1WshrII8tU1Fl6mgmCVhK1tk3v/lNFahctmyZ+jkJzEnwcu+991bb2e1YDfb6Cgm+7bvvvur4ZHqp1D6TIJUEgy655BK1jbyWv/71r/HSSy/ht7/9rbrtwAMPVLdLIKq3tzc9jXeo45Tfi/weOzs71e9y8eLF6nWX10Req91lBsrU4J6eHnVMkt130003qUCiTOmsra1V20hgUAJhEtyUKbaS8fbnP/9ZLTQhQbXs116e5/Tp07Fy5UoVbByKHdA799xzd/s6S8BNAobyOn7ta19TgcGhSNadBCT/9Kc/qe8zyW0SfBwqsEdEREQ0YaSGHBEREeXfqlWrJKXM+sc//mG1tLRYW7dute666y6rurraCgQC1rZt26w33nhDbXPRRRcN+NlvfvOb6vYnn3wyfdvcuXPVbY888sgujxUKhazzzz9/xMd2ySWXqH1lu//++9XtP/zhDwfcfsYZZ1iaplkbN27c7b6PP/54a/Hixeo5y9e7775rff3rX1f7/cQnPqG2ufTSS1X/2WefTf9cT0+PNX/+fGvevHmWYRjqttWrV6vtFixYYEUikQGPc88996j7ZJvRWr58uXo9Mw33+mY/tjjllFPUcWWS34H8LgZ7Tfbdd99dbpfHu/rqq9P98847z9J13Xr55Zd32dY0zSGfz+bNm9W+7L8r24svvqhuv+yyy9K3nXTSSdbSpUutWCw2YN9HHXWUtXDhwl3+fo855hgrlUpZu3PggQda5eXl1kitXLlS7V9er1NPPdX6r//6L+vVV18ddNvPfOYzlt/vt7q6utK3rVu3Tv38lVdeOeLHJCIiIsoVTlklIiIqMFLkXjKMZAqoFPeXBRgk800ylGS1U3H55ZcP+BnJlBPZ0/Rk+uIpp5ySs2OV45EsNsk+yz4eiR89/PDDI9qPTImU5yxfkg323//931i+fHm6ML88jmSCZU5tlNdFssJk+qWsoJpJivpPRK28oV7fzMeW6ZMy5VKmg0rWn/SdINl2slLuJz7xiXTNwUyZ04qHIllu8ndlk9f48MMPT/+dtbe348knn1QrmUomnTwP+Wpra1PPe8OGDSojL5NkCsrfxO7IFNTS0tIRPluolVglw02yCCULU6a6HnzwwTjooINUdmgmmbYqGX9SS84mPys4XZWIiIgKAaesEhERFRip17bXXnupqY4ybVCmQep632doH374ofp+zz33HPAzM2bMUFM75f7sgNFISc0xCcBkkgDZcMEVeTypXZYdWLGnWNrHI9Mv5csm+5R9Z9Zj+81vfqOCSFKjbOHChQPqgsl+JFCULfNxlixZMqbnPR5DPc6//vUvXH311Xj++efV1NFMEpCTRSzGS2rrSVAr83mPlrzO2eRvT6akClnFVAKr/+///T/1NRhZUCEzqDfS115qxEmAcjRkerB8yfOWqdpSt04CbRKUlNp39qIXsjCKTGuV++z6iLJqq6zGKlOJiYiIiPKNATkiIqICI1lKg2U8jTb7SYwmS+y5555TCx5kkoUkJFg2XrJ4gGQ42aSOl2S22aQumWQGOmWiVpId7HGk5t9JJ52k6rnJ6riS6Si13CTrTGrqSWbbZGEfq9TiGyrTMjs4PNLXXl6f119/HVu3blWv0WhIME9WXJUvWS1V6glKgE6yEIXcJll9EuSV+oNbtmxR2Xw//elPR/U4RERERLnCgBwREdEkIoEsCZJIcCGz0L8EHaSwv9w/EoMF9CR7SBZuyM68G2p7+3j+8Y9/qOmMmVlyMgXVvl+cd955A6abjjZgJvuRBRSyZT+OE0HM8ZKFB2QhCVm0YM6cOenbZeELJ0mGoQSmBlsVdaTk7yjbe++9lw7CLliwIB3gcjJgKiSrTbLW/vCHP+DKK68c834keC0BuR07dgy4Xaam3nbbbbj77rtVYFl+/5JdR0RERFQIWEOOiIhoEvm3f/s31d54440DbpdMLCF110ZCMtIkgJepsrJSBV0yv+wpgLK9yP4ZOR7DMHDzzTcPuF0ywSQAIlMH7cBO5n5Hu8qlPI6sRipTQG2ygqesUirBo3322WdEz3mw5+A0e4pv3xoMO6eprlq1ytHHkanLUgNOAoCvvPLKLvdnPv5QpAZdZg04eY0l08z+vcm04Y985CP41a9+tUvAy542O1aySvDSpUvVirKZv1ebBHmlTpyQab+DbSPsOoX2Crc2+RuTvw0J+ElQTrLn6uvrx3y8RERERE5ihhwREdEkIllssmCBBKIksCRBBgmiSIaQBGeyp5wORYrhS2abBPKkBpzU/RqsRlvm9kIWb5CpixJ0kgUnJMtJHlMCJzIFVY7vsccew1//+ldceuml2GOPPRx53t/5zndUNpUEiuQYpD6YPGfJfLr33nvTNfaGc8ABB6jj/slPfqICZD6fDyeeeOKAWnVOWLZsmZqiKq/Nl7/8ZVU7T6ZOyuMMFtQajx/96Efq9Za/A1ngQrIm5THuuece/POf/1R1BYcj000lc/Hiiy9WWX0S6K2ursYVV1wxoKahbCPBM1mwQYKrkpEpAbJt27bhzTffHNOxS9adLLogAdrjjjtOTTGVIJrcvnbtWlX/TYLEErCTgNxRRx2FI444Ah/72MfUFFf5+5eA4rPPPqv+9mWxh0wSEP785z+vXiPx/e9/f0zHSURERJQTOVu/lYiIiEZl1apVktJkvfzyy8Nul0wmre9973vW/PnzLY/HY82ePdu68sorrVgsNmC7uXPnWsuXLx90H+vWrbOOO+44KxAIqMc8//zzh33MVCplfe1rX7OmT59uaZqmfsbW09NjXXbZZVZdXZ06noULF1rXXXedZZrmiJ738ccfb+2Line truncated
|
||||
"text/plain": [
|
||||
"<Figure size 1200x1200 with 9 Axes>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from pathlib import Path\n",
|
||||
"import pandas as pd\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"CSV_ROOT = Path(\"../data/processed\")\n",
|
||||
"dfs = []\n",
|
||||
"\n",
|
||||
"for csv in sorted(CSV_ROOT.rglob(\"*.csv\")):\n",
|
||||
" df = pd.read_csv(\n",
|
||||
" csv,\n",
|
||||
" dtype={\"protocl\": \"uint16\", \"src\": \"uint16\", \"dst\": \"uint16\",\n",
|
||||
" \"classfication\": \"category\"},\n",
|
||||
" engine=\"pyarrow\" # fast if available; falls back otherwise\n",
|
||||
" )\n",
|
||||
" df.rename(columns={\"protocl\": \"protocol\",\n",
|
||||
" \"classfication\": \"classification\"}, inplace=True)\n",
|
||||
" df[\"source\"] = csv.relative_to(CSV_ROOT).as_posix() # tag the rows\n",
|
||||
" dfs.append(df)\n",
|
||||
"\n",
|
||||
"df_all = pd.concat(dfs, ignore_index=True)\n",
|
||||
"print(f\"Loaded {len(df_all):,} packets from {len(dfs)} files\")\n",
|
||||
"\n",
|
||||
"# %% Build a consistent colour map -------------------------------------------\n",
|
||||
"df_all = pd.concat(dfs, ignore_index=True)\n",
|
||||
"\n",
|
||||
"# make sure it's categorical\n",
|
||||
"df_all[\"classification\"] = df_all[\"classification\"].astype(\"category\")\n",
|
||||
"\n",
|
||||
"classes = df_all[\"classification\"].cat.categories\n",
|
||||
"\n",
|
||||
"cmap = {c: plt.cm.tab20(i) for i, c in enumerate(classes)}\n",
|
||||
"\n",
|
||||
"# %% One figure per CSV -------------------------------------------------------\n",
|
||||
"n = len(dfs)\n",
|
||||
"cols = 3 # plots per row\n",
|
||||
"rows = (n + cols - 1) // cols\n",
|
||||
"fig, axes = plt.subplots(rows, cols,\n",
|
||||
" figsize=(cols * 4, rows * 4),\n",
|
||||
" sharex=True, sharey=True)\n",
|
||||
"axes = axes.flatten()\n",
|
||||
"\n",
|
||||
"for ax, (csv, df) in zip(axes, zip(sorted(CSV_ROOT.rglob(\"*.csv\")), dfs)):\n",
|
||||
" # sample up to 200 k rows for speed\n",
|
||||
" df_plot = df if len(df) <= 200_000 else df.sample(200_000, random_state=0)\n",
|
||||
"\n",
|
||||
" for c in classes:\n",
|
||||
" sub = df_plot[df_plot[\"classification\"] == c]\n",
|
||||
" ax.scatter(sub[\"src\"], sub[\"dst\"],\n",
|
||||
" s=4, alpha=0.25, color=cmap[c])\n",
|
||||
"\n",
|
||||
" ax.set_title(csv.stem, fontsize=9)\n",
|
||||
" ax.set_xlim(0, 65535); ax.set_ylim(0, 65535)\n",
|
||||
" ax.set_xlabel(\"src\"); ax.set_ylabel(\"dst\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# hide empty axes if any\n",
|
||||
"for ax in axes[n:]:\n",
|
||||
" ax.axis(\"off\")\n",
|
||||
"\n",
|
||||
"fig.suptitle(\"Port-to-Port Traffic per CSV\")\n",
|
||||
"fig.tight_layout()\n",
|
||||
"plt.legend(classes, loc=\"upper right\", markerscale=4, fontsize=8,\n",
|
||||
" frameon=False, bbox_to_anchor=(1.2, 1.0))"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "cs216",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.13.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
"""diversity_metrics.py (fast version)
|
||||
|
||||
Estimate how much diversity each CSV adds without building a giant in‑memory
|
||||
DataFrame. Designed for IoT packet logs with millions of rows.
|
||||
|
||||
Quick summary printed as a GitHub‑style table (requires *tabulate*; falls back
|
||||
to pandas plain text).
|
||||
|
||||
Usage
|
||||
-----
|
||||
python diversity_metrics.py path/to/processed_dir [-r] [--sample 50000]
|
||||
|
||||
Metrics
|
||||
-------
|
||||
ΔEntropy : change in Shannon entropy of *classification* counts
|
||||
ΔGini : change in Gini impurity of the same counts
|
||||
χ² p : Pearson χ² p‑value old vs new classification counts
|
||||
Jaccard : similarity of unique (src,dst) pairs (0 → new pairs, 1 → no new)
|
||||
KS src p : Kolmogorov–Smirnov p‑value, source‑port dist (uses sampling)
|
||||
KS dst p : Kolmogorov–Smirnov p‑value, dest‑port dist (uses sampling)
|
||||
|
||||
Speed tricks
|
||||
------------
|
||||
* No growing DataFrame; we keep Counters / sets / lists.
|
||||
* Ports for KS are *sampled* (default 50 k) to bound cost.
|
||||
* (src,dst) pairs are hashed to a 32‑bit int to reduce set overhead.
|
||||
* pandas reads via **pyarrow** engine when available.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from collections import Counter
|
||||
from typing import List, Set
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from scipy.stats import chi2_contingency, ks_2samp, entropy
|
||||
|
||||
try:
|
||||
from tabulate import tabulate
|
||||
_USE_TABULATE = True
|
||||
except ImportError:
|
||||
_USE_TABULATE = False
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Helper metrics
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
def shannon(counts: Counter) -> float:
|
||||
total = sum(counts.values())
|
||||
if total == 0:
|
||||
return 0.0
|
||||
p = np.fromiter(counts.values(), dtype=float)
|
||||
p /= total
|
||||
return entropy(p, base=2)
|
||||
|
||||
|
||||
def gini(counts: Counter) -> float:
|
||||
total = sum(counts.values())
|
||||
if total == 0:
|
||||
return 0.0
|
||||
return 1.0 - sum((n / total) ** 2 for n in counts.values())
|
||||
|
||||
|
||||
def jaccard(a: Set[int], b: Set[int]) -> float:
|
||||
if not a and not b:
|
||||
return 1.0
|
||||
return len(a & b) / len(a | b)
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Core analysis
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
def analyse(csv_files: List[Path], sample_size: int):
|
||||
"""Return list of dicts with diversity metrics for each added file."""
|
||||
|
||||
# cumulative state (no big DataFrame!)
|
||||
class_counter: Counter = Counter()
|
||||
pair_hashes: Set[int] = set()
|
||||
src_list: List[int] = []
|
||||
dst_list: List[int] = []
|
||||
|
||||
rows = []
|
||||
|
||||
for csv_path in csv_files:
|
||||
df = pd.read_csv(
|
||||
csv_path,
|
||||
engine="pyarrow" if pd.__version__ >= "2" else "c", # fast parse
|
||||
usecols=["protocl", "src", "dst", "classfication"],
|
||||
dtype={
|
||||
"protocl": "uint16",
|
||||
"protocol": "uint16",
|
||||
"src": "uint16",
|
||||
"dst": "uint16",
|
||||
},
|
||||
)
|
||||
# normalise column names
|
||||
df.rename(columns={"protocl": "protocol", "classfication": "classification"}, inplace=True)
|
||||
|
||||
# snapshot previous state
|
||||
prev_class = class_counter.copy()
|
||||
prev_pairs = pair_hashes.copy()
|
||||
prev_src = np.asarray(src_list, dtype=np.uint16)
|
||||
prev_dst = np.asarray(dst_list, dtype=np.uint16)
|
||||
|
||||
# --- update cumulative structures ------------------------------------
|
||||
class_counter.update(df["classification"].value_counts().to_dict())
|
||||
|
||||
# hash (src,dst) into 32‑bit int to save memory
|
||||
pair_ids = (df["src"].to_numpy(dtype=np.uint32) << np.uint32(16)) | \
|
||||
df["dst"].to_numpy(dtype=np.uint32)
|
||||
|
||||
|
||||
# extend port lists (keep small ints)
|
||||
src_list.extend(df["src"].tolist())
|
||||
dst_list.extend(df["dst"].tolist())
|
||||
|
||||
# --- metrics ----------------------------------------------------------
|
||||
# χ² classification
|
||||
chi_p = np.nan
|
||||
if prev_class:
|
||||
all_classes = list(set(prev_class) | set(df["classification"].unique()))
|
||||
old = [prev_class.get(c, 0) for c in all_classes]
|
||||
new = [df["classification"].value_counts().get(c, 0) for c in all_classes]
|
||||
_, chi_p, _, _ = chi2_contingency([old, new])
|
||||
|
||||
# entropy & gini deltas
|
||||
delta_entropy = shannon(class_counter) - shannon(prev_class)
|
||||
delta_gini = gini(class_counter) - gini(prev_class)
|
||||
|
||||
# Jaccard on pair hashes
|
||||
jc = jaccard(prev_pairs, pair_hashes)
|
||||
|
||||
# KS tests on sampled ports
|
||||
ks_src_p = ks_dst_p = np.nan
|
||||
if prev_src.size:
|
||||
new_src = df["src"].to_numpy(dtype=np.uint16)
|
||||
new_dst = df["dst"].to_numpy(dtype=np.uint16)
|
||||
if prev_src.size > sample_size:
|
||||
prev_src_sample = np.random.choice(prev_src, sample_size, replace=False)
|
||||
else:
|
||||
prev_src_sample = prev_src
|
||||
if new_src.size > sample_size:
|
||||
new_src_sample = np.random.choice(new_src, sample_size, replace=False)
|
||||
else:
|
||||
new_src_sample = new_src
|
||||
if prev_dst.size > sample_size:
|
||||
prev_dst_sample = np.random.choice(prev_dst, sample_size, replace=False)
|
||||
else:
|
||||
prev_dst_sample = prev_dst
|
||||
if new_dst.size > sample_size:
|
||||
new_dst_sample = np.random.choice(new_dst, sample_size, replace=False)
|
||||
else:
|
||||
new_dst_sample = new_dst
|
||||
|
||||
ks_src_p = ks_2samp(prev_src_sample, new_src_sample).pvalue
|
||||
ks_dst_p = ks_2samp(prev_dst_sample, new_dst_sample).pvalue
|
||||
|
||||
rows.append(
|
||||
{
|
||||
"File": csv_path.name,
|
||||
"Rows": len(df),
|
||||
"ΔEntropy": round(delta_entropy, 4),
|
||||
"ΔGini": round(delta_gini, 4),
|
||||
"χ² p": f"{chi_p:.3g}" if not np.isnan(chi_p) else "NA",
|
||||
"Jaccard": round(jc, 3),
|
||||
"KS src p": f"{ks_src_p:.3g}" if not np.isnan(ks_src_p) else "NA",
|
||||
"KS dst p": f"{ks_dst_p:.3g}" if not np.isnan(ks_dst_p) else "NA",
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# CLI
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Evaluate diversity contribution of each CSV (fast version).")
|
||||
ap.add_argument("csv_dir", help="Directory containing CSV files")
|
||||
ap.add_argument("-r", "--recursive", action="store_true", help="Recursively search csv_dir")
|
||||
ap.add_argument("--sample", type=int, default=50_000, help="Sample size for KS tests (default 50k)")
|
||||
args = ap.parse_args()
|
||||
|
||||
root = Path(args.csv_dir)
|
||||
pattern = "**/*.csv" if args.recursive else "*.csv"
|
||||
csv_files = sorted(root.glob(pattern))
|
||||
if not csv_files:
|
||||
print("No CSV files found.")
|
||||
return
|
||||
|
||||
table_rows = analyse(csv_files, args.sample)
|
||||
|
||||
if _USE_TABULATE:
|
||||
print(tabulate(table_rows, headers="keys", tablefmt="github", floatfmt=".4f"))
|
||||
else:
|
||||
print(pd.DataFrame(table_rows).to_string(index=False))
|
||||
|
||||
print(
|
||||
"\nLegend:\n • p-values (χ², KS) < 0.05 → new file significantly shifts distribution (GOOD)"
|
||||
"\n • Positive ΔEntropy or ΔGini → richer mix; near 0 → little new info"
|
||||
"\n • Jaccard close to 0 → many unseen (src,dst) pairs; close to 1 → redundant."
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# Creates the directory layout:
|
||||
# data/
|
||||
# tar/
|
||||
# pcap/
|
||||
# processed/
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
root="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
mkdir -p "$root"/data/{tar,pcap,processed,combined}
|
||||
|
||||
echo "Directory structure ready under $root/data/"
|
||||
Reference in new issue
Block a user