Delete HTML_Similarity.ipynb

HTML_Similarity.ipynb

-{
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "HTML_Similarity.ipynb",
-      "provenance": []
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    }
-  },
-  "cells": [
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "2zXzAV9mRb-L",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "source": [
-        "# -*- coding: utf-8 -*-\n",
-        "\n",
-        "import pandas as pd\n",
-        "from html_similarity import style_similarity, structural_similarity, similarity\n",
-        "from bs4 import BeautifulSoup, Doctype\n",
-        "import imgkit"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "XgqSvWUYRwui",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "source": [
-        "file = pd.read_parquet('data.parquet', engine='pyarrow')\n",
-        "numRows = len(file.index)\n",
-        "\n",
-        "validPayloads = []\n",
-        "timestamps = []\n",
-        "for i in range(numRows):\n",
-        "    payload = file.iloc[i].payload\n",
-        "    mime = file.iloc[i].mime\n",
-        "    timestamp = file.iloc[i].timestamp\n",
-        "\n",
-        "    soup = (BeautifulSoup(payload, \"html.parser\"))\n",
-        "\n",
-        "    # check for only vt.edu\n",
-        "\n",
-        "    if (mime == 'text/html' and len(payload) > 1):\n",
-        "        validPayloads.append(payload)\n",
-        "        timestamps.append(timestamp)\n",
-        "\n"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "Q9UNsU7gRt0Y",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "source": [
-        "for i in range(len(validPayloads)):\n",
-        "    outFileName = 'captures/' + str(timestamps[i]) + '.jpg'\n",
-        "\n",
-        "    #imgkit.from_string(validPayloads[i], outFileName)\n",
-        "\n",
-        "scores = [[-1 for i in range(len(validPayloads))] for j in range(len(validPayloads))]\n",
-        "for i in range(len(validPayloads)):\n",
-        "    payload1 = validPayloads[i]\n",
-        "\n",
-        "    for j in range(len(validPayloads)):\n",
-        "\n",
-        "        payload2 = validPayloads[j]\n",
-        "\n",
-        "        try:\n",
-        "            # print(i,j)\n",
-        "            score = str(similarity(payload1, payload2))\n",
-        "            scores[i][j] = score\n",
-        "            # print(score)\n",
-        "\n",
-        "        except:\n",
-        "            print(i, j)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    }
-  ]
-}
\ No newline at end of file