In [1]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import stats
In [2]:
%matplotlib inline
import warnings
warnings.filterwarnings("ignore")
In [3]:
pd.set_option('display.max_columns',None)
Lets load in our two datasets. One data set is the top trending videos in the united states and the other data set is the comments under the trending videos.¶
In [4]:
US_comments = pd.read_csv('UScomments.csv', error_bad_lines = False)
In [5]:
US_comments.head()
Out[5]:
In [6]:
US_videos = pd.read_csv('USvideos.csv', error_bad_lines = False)
In [7]:
US_videos.head()
Out[7]:
Data exploration and wrangle¶
In [8]:
US_videos.shape
Out[8]:
In [9]:
US_videos.nunique()
Out[9]:
In [10]:
US_videos.info()
In [11]:
US_comments.shape
Out[11]:
In [12]:
US_comments.nunique()
Out[12]:
In [13]:
US_comments.isnull().sum()
Out[13]:
In [14]:
US_comments.dropna(inplace=True)
In [15]:
US_comments.isnull().sum()
Out[15]:
In [16]:
US_comments.shape
Out[16]:
In [17]:
US_comments.nunique()
Out[17]:
In [18]:
US_comments.info()
In [19]:
US_comments.drop(41587, inplace=True)
In [20]:
US_comments.likes = US_comments.likes.astype(int)
US_comments.replies = US_comments.replies.astype(int)
In [21]:
US_comments.head()
Out[21]:
Time for sentiment analysis on US_comments data¶
In [22]:
import nltk
nltk.download('vader_lexicon')
Out[22]:
In [23]:
from nltk.sentiment.vader import SentimentIntensityAnalyzer
sia = SentimentIntensityAnalyzer()
In [24]:
US_comments['Sentiment Scores'] = US_comments['comment_text'].apply(lambda x:sia.polarity_scores(x)['compound'])
In [25]:
US_comments.head()
Out[25]:
In [26]:
US_comments['Sentiment'] = US_comments['Sentiment Scores'].apply(lambda s : 'Positive' if s > 0 else ('Neutral' if s == 0 else 'Negative'))
In [27]:
US_comments.head()
Out[27]:
In [28]:
US_comments.Sentiment.value_counts()
Out[28]:
In [29]:
US_comments.video_id.nunique()
Out[29]:
Number of comments that are positive and creating % of positive comments on videos¶
In [30]:
US_comments[US_comments.video_id == US_comments.video_id.unique()[0]]['Sentiment'].value_counts()
Out[30]:
In [31]:
videos = []
for i in range(0,US_comments.video_id.nunique()):
a = US_comments[(US_comments.video_id == US_comments.video_id.unique()[i]) & (US_comments.Sentiment == 'Positive')].count()[0]
b = US_comments[US_comments.video_id == US_comments.video_id.unique()[i]]['Sentiment'].value_counts().sum()
Percentage = (a/b)*100
videos.append(round(Percentage,2))
Dataframe of the videos with their comments (Use the two data sets to match primary keys of video post/video comments.¶
In [32]:
Positivity = pd.DataFrame(videos,US_comments.video_id.unique()).reset_index()
In [33]:
Positivity.columns = ['video_id','Positive Percentage']
In [34]:
Positivity.head()
Out[34]:
Name the video_id's with their Youtube channels¶
In [35]:
channels = []
for i in range(0,Positivity.video_id.nunique()):
channels.append(US_videos[US_videos.video_id == Positivity.video_id.unique()[i]]['channel_title'].unique())
In [36]:
Positivity['Channel'] = channels
In [37]:
Positivity.head()
Out[37]:
Below shows channels whose comments are marked all positive sentiment¶
In [38]:
Positivity[Positivity['Positive Percentage'] == Positivity['Positive Percentage'].max()]
Out[38]:
Below shows channels who comments were negative (The negative influence channels).¶
In [39]:
Positivity[Positivity['Positive Percentage'] == Positivity['Positive Percentage'].min()]
Out[39]:
Saving files for further analysis and lookup database¶
In [40]:
max_positivity = pd.DataFrame(Positivity[Positivity['Positive Percentage'] == Positivity['Positive Percentage'].max()])
In [41]:
min_positivity = pd.DataFrame(Positivity[Positivity['Positive Percentage'] == Positivity['Positive Percentage'].min()])
In [42]:
min_positivity.to_csv("Worst_Channel_influence.csv") #these are the channels with the worst comments on their videos
In [43]:
max_positivity.to_csv("Best_Channel_influence.csv") #these are the channels with the best comments on their videos
In [44]:
all_channel_sentiment = pd.DataFrame(Positivity[Positivity['Positive Percentage'] == Positivity['Positive Percentage']])
In [45]:
all_channel_sentiment.to_csv("All_Channel_Sentiment.csv")
Lets explore our sentiment values with some data analysis!¶
In [46]:
sns.set_palette('husl')
In [47]:
%pylab inline
In [48]:
import math
In [49]:
data = pd.read_csv('All_Channel_Sentiment_OG.csv')
In [50]:
data.head()
Out[50]:
In [51]:
data.info()
In [52]:
data.describe()
Out[52]:
In [53]:
data["Positive_Percentage"].describe()
Out[53]:
In [54]:
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline
%pylab inline
Distribution of positive percentages across the board¶
In [55]:
sns.set_palette('husl')
plt.figure(figsize=(22.5,10))
sns.distplot(data["Positive_Percentage"], color="skyblue", label="Toxicity Distribution")
Out[55]:
