implement priority aware algorithm,

add dataset size printout

DecisionTree.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 6,
    "id": "d5618056",
    "metadata": {},
    "outputs": [],
@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 7,
    "id": "b96f3403",
    "metadata": {},
    "outputs": [
@@ -30,6 +30,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
+      "dataset size: 4735360\n",
       "train accuracy: 0.879490682862549\n",
       "test accuracy: 0.879490682862549\n"
      ]
@@ -57,6 +58,8 @@
     "Xt = np.array(Xt)\n",
     "Yt = np.array(Yt)\n",
     "\n",
+    "print(f\"dataset size: {len(X)}\")\n",
+    "\n",
     "# decision tree fit\n",
     "dt = DecisionTreeClassifier(max_depth = 5)\n",
     "dt.fit(X, Y)\n",
@@ -69,7 +72,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 8,
    "id": "d336971a",
    "metadata": {},
    "outputs": [],
@@ -134,7 +137,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 9,
    "id": "7f36344d",
    "metadata": {},
    "outputs": [],
@@ -149,21 +152,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
    "id": "cf8832b9",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAB6UAAAYYCAYAAADRlB3IAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzs3XnclnPC///3WWkvW4jQcLUriSyRfQmTsWZfBuPO2BvdjBSyFVlGljGWLGGEwZhhLE1ZI0tJSdtVKksooai0nL8/7u93fvf9nblnzExXR8vz+VeuS+fxqv65zsf7+BxnqVwulwMAAAAAAAAAVaBa0QEAAAAAAAAArL6M0gAAAAAAAABUGaM0AAAAAAAAAFXGKA0AAAAAAABAlTFKAwAAAAAAAFBljNIAAAAAAAAAVBmjNAAAAAAAAABVxigNAAAAAAAAQJUxSgMAAAAAAABQZYzSAAAAAAAAAFQZozQAAAAAAAAAVcYoDQAAAAAAAECVMUoDAAAAAAAAUGWM0gAAAAAAAABUGaM0AAAAAAAAAFXGKA0AAAAAAABAlTFKAwAAAAAAAFBljNIAAAAAAAAAVBmjNAAAAAAAAABVxigNAAAAAAAAQJUxSgMAAAAAAABQZYzSAAAAAAAAAFQZozQAAAAAAAAAVcYoDQAAAAAAAECVMUoDAAAAAAAAUGWM0gAAAAAAAABUGaM0AAAAAAAAAFXGKA0AAAAAAABAlTFKAwAAAAAAAFBljNIAAAAAAAAAVBmjNAAAAAAAAABVxigNAAAAAAAAQJUxSgMAAAAAAABQZYzSAAAAAAAAAFQZozQAAAAAAAAAVcYoDQAAAAAAAECVMUoDAAAAAAAAUGWM0gAAAAAAAABUGaM0AAAAAAAAAFXGKA0AAAAAAABAlTFKAwAAAAAAAFBljNIAAAAAAAAAVBmjNAAAAAAAAABVxigNAAAAAAAAQJUxSgMAAAAAAABQZYzSAAAAAAAAAFQZozQAAAAAAAAAVcYoDQAAAAAAAECVMUoDAAAAAAAAUGWM0gAAAAAAAABUGaM0AAAAAAAAAFXGKA0AAAAAAABAlTFKAwAAAAAAAFBljNIAAAAAAAAAVBmjNAAAAAAAAABVxigNAAAAAAAAQJUxSgMAAAAAAABQZYzSAAAAAAAAAFQZozQAAAAAAAAAVcYoDQAAAAAAAECVMUoDAAAAAAAAUGWM0gAAAAAAAABUGaM0AAAAAAAAAFXGKA0AAAAAAABAlTFKAwAAAAAAAFBljNIAAAAAAAAAVBmjNAAAAAAAAABVxigNAAAAAAAAQJUxSgMAAAAAAABQZYzSAAAAAAAAAFQZozQAAAAAAAAAVcYoDQAAAAAAAECVMUoDAAAAAAAAUGWM0gAAAAAAAABUGaM0AAAAAAAAAFXGKA0AAAAAAABAlTFKAwAAAAAAAFBljNIAAAAAAAAAVBmjNAAAAAAAAABVxigNAAAAAAAAQJUxSgMAAAAAAABQZYzSAAAAAAAAAFQZozQAAAAAAAAAVcYoDQAAAAAAAECVMUoDAAAAAAAAUGWM0gAAAAAAAABUGaM0AAAAAAAAAFXGKA0AAAAAAABAlTFKAwAAAAAAAFBljNIAAAAAAAAAVBmjNAAAAAAAAABVxigNAAAAAAAAQJUxSgMAAAAAAABQZYzSAAAAAAAAAFQZozQAAAAAAAAAVcYoDQAAAAAAAECVMUoDAAAAAAAAUGWM0gAAAAAAAABUGaM0AAAAAAAAAFXGKA0AAAAAAABAlTFKAwAAAAAAAFBljNIAAAAAAAAAVBmjNAAAAAAAAABVxigNAAAAAAAAQJUxSgMAAAAAAABQZYzSAAAAAAAAAFQZozQAAAAAAAAAVcYoDQAAAAAAAECVMUoDAAAAAAAAUGWM0gAAAAAAAABUGaM0AAAAAAAAAFXGKA0AAAAAAABAlTFKAwAAAAAAAFBljNIAAAAAAAAAVBmjNAAAAAAAAABVxigNAAAAAAAAQJUxSgMAAAAAAABQZYzSAAAAAAAAAFQZozQAAAAAAAAAVcYoDQAAAAAAAECVMUoDAAAAAAAAUGWM0gAAAAAAAABUGaM0AAAAAAAAAFXGKA0AAAAAAABAlTFKAwAAAAAAAFBljNIAAAAAAAAAVBmjNAAAAAAAAABVxigNAAAAAAAAQJUxSgMAAAAAAABQZYzSAAAAAAAAAFQZozQAAAAAAAAAVaZG0QEAAAArixkzZmT27NlFZ6wyGjVqlM0337zoDAAAAGAlZ5QGAADIfw3SrVu1yncLFhSdssqoW6dOPpgwwTANAAAA/F1GaQAAgCSzZ8/OdwsW5DfHdUvLjTYoOmelN/GzL9L9wUcze/ZsozQAAADwdxmlAQAA/puWG22Q9ps2WWHXe/DNd7JFo/XToFatfPXdglQrldJsw0Z5Z8ZH2WmLppm/aFE2X2/d3PHK62nbZOMkyYRZn2W9unWzd6sWeXv6zDSqXy/t/s/3AAAAAFY21YoOAAAAWJMdt8N2adygQd6aPiPVSqVM+eKLVM6ekwPbts5X3y3IOnXrZPTMj9Jm443SsHbtfPXdgixZuix1aq6VqbNnZ8+WzQzSAAAAwErNSWkAAIAC3Pv6m2myztqpW7Nmvl6wMFs32STPvj8he7VqnpHTpuezb+ZlzxbNMn/hojRZZ53UXqtG/vje+GzYoH46bfmjzJw7N0MnTM7XCxameqlavlqwIF/Mn59Dt9k6a9epXfQfDwAAAOAvjNIAAAAF+GmnHf7qax2bbpYk2XnLH/3la+vUrfOXXx+7w7Z/+XW7JhvnwLZtqi4QAAAAYDkxSgMAAPw3kz77Ih9/9XXWq1cv8xYuzLhPZqXH3rtnxpdz06BWrUz5YnZaNt4wr06Zms/nzc+GDepn/Xr1MvnzL/L90qVp3XijLF22LGvXqZNZ33yTDpttms++mZdZ38zL9C+/zNp16qRxgwb5euGCTPzsi7TZeKOsX69eqpVKmb9oUdpusnE+mvtV5i1alGmz56RWjbWyydoNM/vbbzPli9n50frrpcWGG2TeokUpl8vp2HSzvDB+YjpsvmlemTI1CxcvzqHbbJ3XKqemccOG2XbzTXPbS69lm82apHqplMmff5HGDRumzlprZWl5Wd6aPjP/0blTfjd6TPZq2TyNGzbIA2++k0b16mXLDRplxpdfZuzHn2bX5hUpLyunTs218t3336dB7VpF/1MBAAAAqwijNAAAwH/TYqMN0n7TJn/5731bt0ySbL7eunlqzLhsvHbDjJrxUWZ++VWO3WG7/G70mLTdZONsu/mmufpPQ9Nuk42zbFk5M76cmy/mz8/ipUuzYYMGWbdunaxVvVHabrJxZnw5Nx9//XUa1auXj7/6Oltt3DjTv5ybDz79LB022zSjZn6UvVo2zy4VW+Tht0Zn1rx5abPxRqlbc6189d3CfPL1138ZskdOm56dK7bI29NnplqplAXfL87wiZOz7eabZvLns3P90Bf/x+i8fdPN8tR77ydJ6qy1Vlo33jB3vfZG2my8UT6fNz+z53+bFhtukM7NtkyStN2k8d88kT3mo49XwL8GAAAAsDoolcvlctERAAAARRs1alS22267vPiLM/7HKM3fNuajj7PHDbflnXfeybbbbvuPfwMAAACwxnJSGgAA4Ad48t2x2WSdtTN7/vx8Pm9+Nl67YdauUyfzFi5M68YbZeSHM7J784rM+mZeZnw5N7O++SY/Wn+91KpeI9PmzMlBW7fNw2+NyuHbtk/D2rXyh7Hjs2xZOU3WXTvlcjlvTJv+vz6ae7cWFXl1ytTM+mZedvhR03z93YLM/OqrNKxdK03XWy+fz5ufmtWrZ5dmW2TQiJHZZtMmmfXNN/lg1uf/45R0k3XWzm9eeT2br7tOftRo/cz4cm4+/fq/OmvXqJHpX87NwiWLs1/rlpn8+ew0ql8vCxcvzvY/2rzov34AAABgFeakNAAAQP7/k9K/Oa5bWm60QdE5K4VFS5Zk7ncL0rhhg7/63sTPvkj3Bx91UhoAAAD4h5yUBgAASNKoUaPUrVMn3R98tOiUVUbdOnXSqFGjojMAAACAlZyT0gAAAP/HjBkzMnv27OX+ulOmTMktt9ySV155Ja1bt84555yTHXbYYblf5/vvv89jjz2Wu+66KwsXLsyxxx6bk046KQ0a/PVJ5+WhUaNG2Xxzj/YGAAAA/j6jNAAAQBWZOXNmLr300tx3333ZYostcvXVV+eII45ItWrVqvS6X3/9dQYMGJAbbrghderUSe/evXPGGWekVq1aVXpdAAAAgL/FKA0AALCczZ07N/3798/AgQPToEGDXHrppTnttNNSs2bNFdrxySefpG/fvrn77ruz6aab5sorr8yxxx5b5aM4AAAAwH9nlAYAAFhOFi5cmFtuuSVXX311vv/++5x//vnp2bNnlT0++4eaMGFCevXqlSeeeCLt27fPNddck/322y+lUqnQLgAAAGDN4PZ4AACAf9PSpUtz3333pUWLFvnlL3+Zo48+OlOmTEnfvn0LH6STpFWrVnn88cfz2muvpX79+tl///2z77775p133ik6DQAAAFgDGKUBAAD+ReVyOc8880w6dOiQn/70p9lxxx0zfvz43HbbbWncuHHReX9l5513ziuvvJLf//73+eSTT9KxY8ccc8wxmTp1atFpAAAAwGrMKA0AAPAvePPNN7PXXnvlxz/+cdZbb7288cYbefTRR9OiRYui0/6uUqmUn/zkJ3nvvfdy11135eWXX06rVq1yzjnn5Isvvig6DwAAAFgNGaUBAAD+CZMnT86RRx6ZHXfcMbNnz87TTz+d4cOHZ8cddyw67Z9So0aNnHrqqZk8eXIuv/zy3H///amoqMgVV1yRb7/9tug8AAAAYDVilAYAAPgBPvvss5x55plp06ZN3njjjdx777159913c+CBB6ZUKhWd9y+rW7dufvnLX6aysjKnnXZarrzyyjRr1iy33357Fi9Line truncated
-      "text/plain": [
-       "<Figure size 2500x2000 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "fig = plt.figure(figsize=(25,20))\n",
     "_ = plot_tree(dt, filled=True)"

TreeToRMT.ipynb

@@ -103,7 +103,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "0dc1d6d4",
    "metadata": {},
    "outputs": [
@@ -117,8 +117,8 @@
       "[1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
       "id mapping: \n",
       "[['dst_range', 'dst_meta'], ['src_range', 'src_meta'], ['protocl_range', 'protocl_meta'], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]\n",
-      "13312\n",
-      "110\n"
+      "TCAM bits: 13312\n",
+      "RAM bits:  110\n"
      ]
     }
    ],
@@ -204,6 +204,7 @@
     "\tfor layer in layers:\n",
     "\t\tnum_prefixes = 0\n",
     "\t\tprefix_width = field_width[layer]\n",
+    "\t\t# for each range in the layer, convert the ranges to prefixes using naive range expansion\n",
     "\t\tfor r in layers[layer]:\n",
     "\t\t\tif r[\"min\"] == None:\n",
     "\t\t\t\tr[\"min\"] = 0\n",
@@ -250,7 +251,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "id": "48011528",
    "metadata": {},
    "outputs": [
@@ -264,8 +265,8 @@
       "[1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
       "id mapping: \n",
       "[['dst_range', 'dst_meta'], ['src_range', 'src_meta'], ['protocl_range', 'protocl_meta'], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]\n",
-      "3520\n",
-      "110\n"
+      "TCAM bits: 3520\n",
+      "RAM bits:  110\n"
      ]
     }
    ],
@@ -274,6 +275,111 @@
     "print(f\"TCAM bits: {tcam_bits}\")\n",
     "print(f\"RAM bits:  {ram_bits}\")"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "64b7271e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# for this technique, we note that given disjoint ranges [0,a][a,b],[b,c] ...\n",
+    "# then if using a TCAM that selects the first matching prefix, then [0,a],[0,b],[0,c] would be equivalent\n",
+    "# this is because if for some k<a, even though the range [0,b] could be selected, as long as the prefixes for [0,a] are before [0,b] then the correct prefix will still be selected\n",
+    "\n",
+    "def priority_aware(tree):\n",
+    "\trmt = []\n",
+    "\tstep = 0\n",
+    "\n",
+    "\ttcam_bits = 0\n",
+    "\tram_bits = 0\n",
+    "\n",
+    "\tfor layer in layers:\n",
+    "\t\tnum_prefixes = 0\n",
+    "\t\tprefix_width = field_width[layer]\n",
+    "\t\t# for each range, run the regular prefix expansion, and also the prefix expansion setting the minimum to 0\n",
+    "\t\t# then check which set of prefixes would be better\n",
+    "\t\t# we will assume the ranges are already disjoin and in the correct order\n",
+    "\t\tfor r in layers[layer]:\n",
+    "\t\t\tif r[\"min\"] == None:\n",
+    "\t\t\t\tr[\"min\"] = 0\n",
+    "\t\t\telif r[\"max\"] == None:\n",
+    "\t\t\t\tr[\"max\"] = 2 ** prefix_width\n",
+    "\t\t\tregular_prefixes = convert_range(r[\"min\"], r[\"max\"], prefix_width)\n",
+    "\t\t\tzero_start_prefixes = convert_range(0, r[\"max\"], prefix_width)\n",
+    "\n",
+    "\t\t\tif len(regular_prefixes) <= len(zero_start_prefixes):\n",
+    "\t\t\t\tpfx_type = \"exact\"\n",
+    "\t\t\t\tprefixes = regular_prefixes\n",
+    "\t\t\telse:\n",
+    "\t\t\t\tpfx_type = \"zero\"\n",
+    "\t\t\t\tprefixes = zero_start_prefixes\n",
+    "\n",
+    "\t\t\tr[\"prefixes\"] = prefixes\n",
+    "\t\t\tr[\"prefix_type\"] = pfx_type\n",
+    "\t\t\tnum_prefixes += len(prefixes)\n",
+    "\t\t\ttcam_bits += len(prefixes) * prefix_width\n",
+    "\n",
+    "\t\ttcam = {\n",
+    "\t\t\t\"id\": f\"{layer}_range\",\n",
+    "\t\t\t\"step\": step,\n",
+    "\t\t\t\"match\": \"ternary\",\n",
+    "\t\t\t\"entries\": num_prefixes,\n",
+    "\t\t\t\"key_size\": prefix_width,\n",
+    "\t\t\t\"ranges\": layers[layer]\n",
+    "\t\t}\n",
+    "\n",
+    "\t\tnum_ranges = len(layers[layer])\n",
+    "\t\t# assume no pointer reuse for metadata storage\n",
+    "\t\tram = {\n",
+    "\t\t\t\"id\": f\"{layer}_meta\",\n",
+    "\t\t\t\"step\": step,\n",
+    "\t\t\t\"match\": \"exact\",\n",
+    "\t\t\t\"method\": \"index\",\n",
+    "\t\t\t\"key_size\": math.ceil(math.log2(num_ranges)),\n",
+    "\t\t\t\"data_size\": len(classes)\n",
+    "\t\t}\n",
+    "\t\tram_bits += math.ceil(math.log2(num_ranges)) * len(classes)\n",
+    "\n",
+    "\t\trmt.append(tcam)\n",
+    "\t\trmt.append(ram)\n",
+    "\n",
+    "\t\tstep += 1\n",
+    "\n",
+    "\treturn rmt, tcam_bits, ram_bits\n",
+    "\n",
+    "x, tcam_bits, ram_bits = priority_aware(tree)\n",
+    "f = open(\"priority_aware.json\", \"w+\")\n",
+    "f.write(json.dumps(x, indent=4))\n",
+    "f.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "cd706e41",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "TCAM mapping: \n",
+      "[1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
+      "SRAM mapping: \n",
+      "[1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
+      "id mapping: \n",
+      "[['dst_range', 'dst_meta'], ['src_range', 'src_meta'], ['protocl_range', 'protocl_meta'], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]\n",
+      "TCAM bits: 2120\n",
+      "RAM bits:  110\n"
+     ]
+    }
+   ],
+   "source": [
+    "! command python3 ideal-rmt-simulator/sim.py priority_aware.json\n",
+    "print(f\"TCAM bits: {tcam_bits}\")\n",
+    "print(f\"RAM bits:  {ram_bits}\")"
+   ]
   }
  ],
  "metadata": {