long term using linear regression and forecast using arima.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ebf7bac1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from statsmodels.tsa.arima.model import ARIMA\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.linear_model import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "1bc562ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv(r\"C:\\Users\\Nidhi\\Desktop\\myntra\\dataverse_files\\fashion data on Instagram.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "e6d5f322",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserId</th>\n",
       "      <th>Followings</th>\n",
       "      <th>Followers</th>\n",
       "      <th>MediaCount</th>\n",
       "      <th>BrandName</th>\n",
       "      <th>BrandCategory</th>\n",
       "      <th>Hashtags</th>\n",
       "      <th>Caption</th>\n",
       "      <th>ImgURL</th>\n",
       "      <th>Likes</th>\n",
       "      <th>...</th>\n",
       "      <th>NumberOfPeople</th>\n",
       "      <th>NumberOfFashionProduct</th>\n",
       "      <th>Anger</th>\n",
       "      <th>Contempt</th>\n",
       "      <th>Disgust</th>\n",
       "      <th>Fear</th>\n",
       "      <th>Happiness</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Sadness</th>\n",
       "      <th>Surprise</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1171579752855683619_212070047</td>\n",
       "      <td>518.0</td>\n",
       "      <td>9840.0</td>\n",
       "      <td>541</td>\n",
       "      <td>abercrombie</td>\n",
       "      <td>High street</td>\n",
       "      <td>beautiful, summer, fashion, love, cute, food, ...</td>\n",
       "      <td>We were born to be REAL, not to be perfect. ...</td>\n",
       "      <td>https://scontent.cdninstagram.com/t51.2885-15/...</td>\n",
       "      <td>97</td>\n",
       "      <td>...</td>\n",
       "      <td>0.931486</td>\n",
       "      <td>3.42211</td>\n",
       "      <td>0.083862</td>\n",
       "      <td>0.015089</td>\n",
       "      <td>0.000583</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>0.890586</td>\n",
       "      <td>0.009657</td>\n",
       "      <td>0.000181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1171594777274371222_176762322</td>\n",
       "      <td>7333.0</td>\n",
       "      <td>2300.0</td>\n",
       "      <td>272</td>\n",
       "      <td>abercrombie</td>\n",
       "      <td>High street</td>\n",
       "      <td>teen, model, brunette, selfie, hollister, snap...</td>\n",
       "      <td>Gotta run, but first, let me take selfie. Me...</td>\n",
       "      <td>https://scontent.cdninstagram.com/t51.2885-15/...</td>\n",
       "      <td>94</td>\n",
       "      <td>...</td>\n",
       "      <td>0.466329</td>\n",
       "      <td>2.91971</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1171407552643586413_581125501</td>\n",
       "      <td>131.0</td>\n",
       "      <td>605.0</td>\n",
       "      <td>106</td>\n",
       "      <td>abercrombie</td>\n",
       "      <td>High street</td>\n",
       "      <td>alexandani, cute, llbean, beanboots, anthropol...</td>\n",
       "      <td>dress to impress\\U0001f457</td>\n",
       "      <td>https://scontent.cdninstagram.com/t51.2885-15/...</td>\n",
       "      <td>91</td>\n",
       "      <td>...</td>\n",
       "      <td>1.116500</td>\n",
       "      <td>4.69096</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1171407552643586413_581125501</td>\n",
       "      <td>131.0</td>\n",
       "      <td>605.0</td>\n",
       "      <td>106</td>\n",
       "      <td>abercrombie</td>\n",
       "      <td>High street</td>\n",
       "      <td>alexandani, cute, llbean, beanboots, anthropol...</td>\n",
       "      <td>dress to impress\\U0001f457</td>\n",
       "      <td>https://scontent.cdninstagram.com/t51.2885-15/...</td>\n",
       "      <td>94</td>\n",
       "      <td>...</td>\n",
       "      <td>1.116500</td>\n",
       "      <td>4.69096</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1171508187966229230_2797323089</td>\n",
       "      <td>132.0</td>\n",
       "      <td>456.0</td>\n",
       "      <td>56</td>\n",
       "      <td>abercrombie</td>\n",
       "      <td>High street</td>\n",
       "      <td>abercrombieandfitch, shopping, love, hollister...</td>\n",
       "      <td>#love #shopping #shoppen #hollister #abercro...</td>\n",
       "      <td>https://scontent.cdninstagram.com/t51.2885-15/...</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>0.393063</td>\n",
       "      <td>1.45586</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                           UserId  Followings  Followers  MediaCount  \\\n",
       "0   1171579752855683619_212070047       518.0     9840.0         541   \n",
       "1   1171594777274371222_176762322      7333.0     2300.0         272   \n",
       "2   1171407552643586413_581125501       131.0      605.0         106   \n",
       "3   1171407552643586413_581125501       131.0      605.0         106   \n",
       "4  1171508187966229230_2797323089       132.0      456.0          56   \n",
       "\n",
       "     BrandName BrandCategory  \\\n",
       "0  abercrombie   High street   \n",
       "1  abercrombie   High street   \n",
       "2  abercrombie   High street   \n",
       "3  abercrombie   High street   \n",
       "4  abercrombie   High street   \n",
       "\n",
       "                                            Hashtags  \\\n",
       "0  beautiful, summer, fashion, love, cute, food, ...   \n",
       "1  teen, model, brunette, selfie, hollister, snap...   \n",
       "2  alexandani, cute, llbean, beanboots, anthropol...   \n",
       "3  alexandani, cute, llbean, beanboots, anthropol...   \n",
       "4  abercrombieandfitch, shopping, love, hollister...   \n",
       "\n",
       "                                             Caption  \\\n",
       "0    We were born to be REAL, not to be perfect. ...   \n",
       "1    Gotta run, but first, let me take selfie. Me...   \n",
       "2                        dress to impress\\U0001f457    \n",
       "3                        dress to impress\\U0001f457    \n",
       "4    #love #shopping #shoppen #hollister #abercro...   \n",
       "\n",
       "                                              ImgURL  Likes  ...  \\\n",
       "0  https://scontent.cdninstagram.com/t51.2885-15/...     97  ...   \n",
       "1  https://scontent.cdninstagram.com/t51.2885-15/...     94  ...   \n",
       "2  https://scontent.cdninstagram.com/t51.2885-15/...     91  ...   \n",
       "3  https://scontent.cdninstagram.com/t51.2885-15/...     94  ...   \n",
       "4  https://scontent.cdninstagram.com/t51.2885-15/...      9  ...   \n",
       "\n",
       "   NumberOfPeople  NumberOfFashionProduct     Anger  Contempt   Disgust  \\\n",
       "0        0.931486                 3.42211  0.083862  0.015089  0.000583   \n",
       "1        0.466329                 2.91971  0.000000  0.000000  0.000000   \n",
       "2        1.116500                 4.69096  0.000000  0.000000  0.000000   \n",
       "3        1.116500                 4.69096  0.000000  0.000000  0.000000   \n",
       "4        0.393063                 1.45586  0.000000  0.000000  0.000000   \n",
       "\n",
       "       Fear  Happiness   Neutral   Sadness  Surprise  \n",
       "0  0.000026   0.000017  0.890586  0.009657  0.000181  \n",
       "1  0.000000   0.000000  0.000000  0.000000  0.000000  \n",
       "2  0.000000   0.000000  0.000000  0.000000  0.000000  \n",
       "3  0.000000   0.000000  0.000000  0.000000  0.000000  \n",
       "4  0.000000   0.000000  0.000000  0.000000  0.000000  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "d5d3c569",
   "metadata": {},
   "outputs": [],
   "source": [
    "# converting unix dates to yyyy-mm-dd\n",
    "df['Date'] = pd.to_datetime(df['CreationTime'], unit='s').dt.strftime('%Y-%m-%d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "21f81f68",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "abb741bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0        [beautiful, summer, fashion, love, cute, food,...\n",
      "1        [teen, model, brunette, selfie, hollister, sna...\n",
      "2        [alexandani, cute, llbean, beanboots, anthropo...\n",
      "3        [alexandani, cute, llbean, beanboots, anthropo...\n",
      "4        [abercrombieandfitch, shopping, love, holliste...\n",
      "                               ...                        \n",
      "24747    [เสื้อเท่ๆ, เสื้อใส่สบาย, เสื้อไปทะเล, เสื้อสว...\n",
      "24748    [siambrandname, topshopthailand, somethingboud...\n",
      "24749    [siambrandname, coach, peony, zara, รับตัวแทน,...\n",
      "24750    [siambrandname, coach, peony, zara, รับตัวแทน,...\n",
      "24751    [siambrandname, topshopthailand, somethingboud...\n",
      "Name: Hashtags, Length: 24589, dtype: object\n"
     ]
    }
   ],
   "source": [
    "#convert encoded bytes to readable\n",
    "def decode_unicode(text):\n",
    "    return bytes(text, 'utf-8').decode('unicode_escape')\n",
    "\n",
    "df['Hashtags'] = df['Hashtags'].str.split(', ').apply(lambda tags: [decode_unicode(tag) for tag in tags])\n",
    "\n",
    "print(df['Hashtags'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "d730521e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#get useful columns\n",
    "df_required = df[['Date', 'Hashtags']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "a3642b2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# separate hashtags\n",
    "df_exploded = df_required.explode('Hashtags')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "153db9d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Nidhi\\AppData\\Local\\Temp\\ipykernel_22944\\2866156661.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_required['Date'] = pd.to_datetime(df_required['Date'])\n"
     ]
    }
   ],
   "source": [
    "# get date in datetime format\n",
    "df_required['Date'] = pd.to_datetime(df_required['Date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "849c2730",
   "metadata": {},
   "outputs": [],
   "source": [
    "# separate hashtags\n",
    "df_exploded = df_required.explode('Hashtags')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "25845831",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Nidhi\\AppData\\Local\\Temp\\ipykernel_22944\\2866156661.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_required['Date'] = pd.to_datetime(df_required['Date'])\n"
     ]
    }
   ],
   "source": [
    "# get date in datetime format\n",
    "df_required['Date'] = pd.to_datetime(df_required['Date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "445d26ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_exploded=df_exploded.set_index(['Date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "e86a6d64",
   "metadata": {},
   "outputs": [],
   "source": [
    "# groupby weekly trends\n",
    "hashtag_trends = df_exploded.groupby('Hashtags').resample('W-Mon').size().reset_index(name='Count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "ec68571f",
   "metadata": {},
   "outputs": [],
   "source": [
    "hashtag_trends=hashtag_trends.set_index(['Date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "ac1d2834",
   "metadata": {},
   "outputs": [],
   "source": [
    "hashtag_trends_pivot = hashtag_trends.pivot(columns='Hashtags', values='Count').fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "b12008a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter to get dates after 2015\n",
    "filtered_hashtag_trends_pivot = hashtag_trends_pivot[hashtag_trends_pivot.index >= \"2015-01-01\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "642a1e3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_long_term = filtered_hashtag_trends_pivot.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "d0685ce4",
   "metadata": {},
   "outputs": [],
   "source": [
    "hashtag_slopes = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "a628b629",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Fit linear regression and calculate slopes\n",
    "for hashtag in df_long_term.columns:\n",
    "    y = df_long_term[hashtag].values.reshape(-1, 1)\n",
    "    X = np.arange(len(df_long_term)).reshape(-1, 1)  # Use indices as X for simplicity\n",
    "    model = LinearRegression()\n",
    "    model.fit(X, y)\n",
    "    slope = model.coef_[0][0]  # Slope of the linear regression line\n",
    "    hashtag_slopes[hashtag] = slope"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "56267d79",
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_hashtag_slopes = sorted(hashtag_slopes.items(), key=lambda x: x[1], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "24d532f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top Trending Hashtags (Long-term):\n",
      "fashion: Slope=8.61\n",
      "chanel: Slope=5.01\n",
      "ootd: Slope=4.60\n",
      "style: Slope=4.25\n",
      "prada: Slope=3.78\n",
      "dior: Slope=3.62\n",
      "gucci: Slope=3.31\n",
      "hermes: Slope=3.26\n",
      "zara: Slope=2.98\n",
      "louisvuitton: Slope=2.66\n"
     ]
    }
   ],
   "source": [
    "# Display top trending hashtags based on slopes\n",
    "print(\"Top Trending Hashtags (Long-term):\")\n",
    "for tag, slope in sorted_hashtag_slopes[:10]:  # Displaying top 10 for brevity\n",
    "    print(f\"{tag}: Slope={slope:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5fcd4b16",
   "metadata": {},
   "source": [
    "using arima forecasting the hashtags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "d914a508",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter the data for the top trending hashtags\n",
    "top_hashtags = ['fashion', 'chanel', 'ootd', 'style', 'prada', 'dior', 'gucci', 'hermes', 'zara', 'louisvuitton']\n",
    "df_filtered = df_long_term[top_hashtags]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "df57033b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def forecast_hashtag(df, hashtag, forecast_steps=13, order=(1, 1, 1)):\n",
    "    series = df[hashtag]\n",
    "    data = series.values.astype(float)\n",
    "    \n",
    "    # Fit ARIMA model\n",
    "    model = ARIMA(data, order=order)\n",
    "    model_fit = model.fit()\n",
    "    \n",
    "    # Forecast\n",
    "    forecast = model_fit.forecast(steps=forecast_steps)\n",
    "    \n",
    "    return forecast\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "668882b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Nidhi\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\statsmodels\\tsa\\statespace\\sarimax.py:966: UserWarning: Non-stationary starting autoregressive parameters found. Using zeros as starting parameters.\n",
      "  warn('Non-stationary starting autoregressive parameters'\n",
      "C:\\Users\\Nidhi\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\statsmodels\\tsa\\statespace\\sarimax.py:978: UserWarning: Non-invertible starting MA parameters found. Using zeros as starting parameters.\n",
      "  warn('Non-invertible starting MA parameters found.'\n"
     ]
    }
   ],
   "source": [
    "forecasts = {}\n",
    "\n",
    "for hashtag in top_hashtags:\n",
    "    forecasts[hashtag] = forecast_hashtag(df_filtered, hashtag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "3c845f10",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The hashtag with the highest mean forecast is 'zara' with a mean forecast of 15896.43 over next 13 weeks\n"
     ]
    }
   ],
   "source": [
    "# Calculate mean forecast for each hashtag\n",
    "mean_forecasts = {hashtag: sum(values) / len(values) for hashtag, values in forecasts.items()}\n",
    "\n",
    "# Sort mean forecasts by value (descending)\n",
    "sorted_mean_forecasts = sorted(mean_forecasts.items(), key=lambda x: x[1], reverse=True)\n",
    "\n",
    "# Print the hashtag with the highest mean forecast\n",
    "highest_hashtag, highest_mean_forecast = sorted_mean_forecasts[0]\n",
    "print(f\"The hashtag with the highest mean forecast is '{highest_hashtag}' with a mean forecast of {highest_mean_forecast:.2f} over next 13 weeks\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5c70763",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}