basic classifier

9664f130 · Naman Ahuja · 6c107512 · 9664f130
Commit 9664f130 authored 5 years ago by Naman Ahuja
--- a/ExampleNotebooks/contentClassifier.ipynb
+++ b/ExampleNotebooks/contentClassifier.ipynb
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import pandas as pd\n",
+    "from html_similarity import style_similarity, structural_similarity, similarity\n",
+    "from bs4 import BeautifulSoup, Doctype"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "root = 'data'\n",
+    "modelUrl = 'vt.edu'\n",
+    "\n",
+    "parquetFiles = []\n",
+    "for root, dirs, files in os.walk(root):\n",
+    "    path = root.split(os.sep)\n",
+    "    for file in files:\n",
+    "        if file.endswith(\".parquet\"):\n",
+    "            parquetFiles.append(os.path.join(root, file))\n",
+    "\n",
+    "parquetFiles.sort()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/naman/anaconda3/envs/archive/lib/python3.7/site-packages/pyarrow/pandas_compat.py:752: FutureWarning: .labels was deprecated in version 0.24.0. Use .codes instead.\n",
+      "  labels, = index.labels\n"
+     ]
+    }
+   ],
+   "source": [
+    "archiveData = []\n",
+    "\n",
+    "for fileName in parquetFiles:\n",
+    "    file = pd.read_parquet(fileName, engine='pyarrow')\n",
+    "    numRows = len(file.index)\n",
+    "    \n",
+    "    for i in range(numRows):\n",
+    "        UriComponents = file.iloc[i].originalUrl.split('/')\n",
+    "        payload = file.iloc[i].payload\n",
+    "        mime = file.iloc[i].mime\n",
+    "        timestamp = file.iloc[i].timestamp\n",
+    "        soup = (BeautifulSoup(payload, \"html.parser\"))\n",
+    "\n",
+    "        if (mime == 'text/html' and len(payload) > 1 and modelUrl in UriComponents[-2]):\n",
+    "            currentData = {}\n",
+    "            currentData['payload'] = payload\n",
+    "            currentData['timestamp'] = timestamp\n",
+    "            archiveData.append(currentData)\n",
+    "\n",
+    "archiveData.sort(key=lambda x: x['timestamp'], reverse=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "basePayload = archiveData[0]['payload']\n",
+    "lastSavedDataIndex = 0\n",
+    "dataset = []\n",
+    "\n",
+    "for i in range(1, len(archiveData)):\n",
+    "    hasContentChanged = 0\n",
+    "    \n",
+    "    overallSimilarity = similarity(basePayload, archiveData[i]['payload'])\n",
+    "    styleSimilarity = style_similarity(basePayload, archiveData[i]['payload'])\n",
+    "    structuralSimilarity = structural_similarity(basePayload, archiveData[i]['payload'])\n",
+    "    \n",
+    "    if(overallSimilarity < 0.80):\n",
+    "        hasContentChanged = 1\n",
+    "        lastSavedDataIndex = i\n",
+    "        basePayload = archiveData[i]['payload']\n",
+    "    \n",
+    "    data = [overallSimilarity, styleSimilarity, structuralSimilarity, hasContentChanged]\n",
+    "    dataset.append(data)\n",
+    "\n",
+    "df = pd.DataFrame(dataset, columns = ['similarity', 'styleSimilarity', 'styleSimilarity', 'changed']) \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>similarity</th>\n",
+       "      <th>styleSimilarity</th>\n",
+       "      <th>styleSimilarity</th>\n",
+       "      <th>changed</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>0</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>0.381462</td>\n",
+       "      <td>0.242424</td>\n",
+       "      <td>0.520499</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3</td>\n",
+       "      <td>0.383244</td>\n",
+       "      <td>0.242424</td>\n",
+       "      <td>0.524064</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   similarity  styleSimilarity  styleSimilarity  changed\n",
+       "0    1.000000         1.000000         1.000000        0\n",
+       "1    1.000000         1.000000         1.000000        0\n",
+       "2    0.381462         0.242424         0.520499        1\n",
+       "3    0.383244         0.242424         0.524064        1"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
+%% Cell type:code id: tags:
+
+``` python
+import os
+import pandas as pd
+from html_similarity import style_similarity, structural_similarity, similarity
+from bs4 import BeautifulSoup, Doctype
+```
+
+%% Cell type:code id: tags:
+
+``` python
+root = 'data'
+modelUrl = 'vt.edu'
+
+parquetFiles = []
+for root, dirs, files in os.walk(root):
+    path = root.split(os.sep)
+    for file in files:
+        if file.endswith(".parquet"):
+            parquetFiles.append(os.path.join(root, file))
+
+parquetFiles.sort()
+```
+
+%% Cell type:code id: tags:
+
+``` python
+archiveData = []
+
+for fileName in parquetFiles:
+    file = pd.read_parquet(fileName, engine='pyarrow')
+    numRows = len(file.index)
+
+    for i in range(numRows):
+        UriComponents = file.iloc[i].originalUrl.split('/')
+        payload = file.iloc[i].payload
+        mime = file.iloc[i].mime
+        timestamp = file.iloc[i].timestamp
+        soup = (BeautifulSoup(payload, "html.parser"))
+
+        if (mime == 'text/html' and len(payload) > 1 and modelUrl in UriComponents[-2]):
+            currentData = {}
+            currentData['payload'] = payload
+            currentData['timestamp'] = timestamp
+            archiveData.append(currentData)
+
+archiveData.sort(key=lambda x: x['timestamp'], reverse=False)
+```
+
+%% Output
+
+    /home/naman/anaconda3/envs/archive/lib/python3.7/site-packages/pyarrow/pandas_compat.py:752: FutureWarning: .labels was deprecated in version 0.24.0. Use .codes instead.
+      labels, = index.labels
+
+%% Cell type:code id: tags:
+
+``` python
+basePayload = archiveData[0]['payload']
+lastSavedDataIndex = 0
+dataset = []
+
+for i in range(1, len(archiveData)):
+    hasContentChanged = 0
+
+    overallSimilarity = similarity(basePayload, archiveData[i]['payload'])
+    styleSimilarity = style_similarity(basePayload, archiveData[i]['payload'])
+    structuralSimilarity = structural_similarity(basePayload, archiveData[i]['payload'])
+
+    if(overallSimilarity < 0.80):
+        hasContentChanged = 1
+        lastSavedDataIndex = i
+        basePayload = archiveData[i]['payload']
+
+    data = [overallSimilarity, styleSimilarity, structuralSimilarity, hasContentChanged]
+    dataset.append(data)
+
+df = pd.DataFrame(dataset, columns = ['similarity', 'styleSimilarity', 'styleSimilarity', 'changed'])
+```
+
+%% Cell type:code id: tags:
+
+``` python
+df
+```
+
+%% Output
+
+       similarity  styleSimilarity  styleSimilarity  changed
+    0    1.000000         1.000000         1.000000        0
+    1    1.000000         1.000000         1.000000        0
+    2    0.381462         0.242424         0.520499        1
+    3    0.383244         0.242424         0.524064        1
+
+%% Cell type:code id: tags:
+
+``` python
+```