数据来源为kaggle：https://www.kaggle.com/jameslko/gun-violence-data
主要分析13-18年美国枪支暴力事件的特征，以及使用时间序列预测下一年枪支暴力事件发生数量。
主要用到以下数据包
Basemap是python中一个利用地图的库
plotly是开挂的作图神器，可以供js, python, R, DB等使用
Seaborn是基于matplotlib的python数据可视化库，提供更高层次的API封装，使用起来更加方便快捷。
Fbprophet ：facebook开源的时间序列预测框架prophet，目前支持R语言和python语言。托管在github上：https://github.com/facebookincubator/prophet。
wordcloud 生成词云
nltk.corpus 自带词袋中的停用词表去英文停用词
#数据基础操作 import pandas as pd # package for high-performance, easy-to-use data structures and data analysis import numpy as np # fundamental package for scientific computing with Python #画图 import matplotlib import matplotlib.pyplot as plt # for plotting import seaborn as sns # for making plots with seaborn color = sns.color_palette() from plotly.offline import init_notebook_mode, iplot,plot init_notebook_mode(connected=True) import plotly.graph_objs as go import squarify from numpy import array from matplotlib import cm ''' 数据处理包 ''' from sklearn import preprocessing import warnings warnings.filterwarnings("ignore") from nltk.corpus import stopwords from textblob import TextBlob import datetime as dt import warnings import string import time stop_words = list(set(stopwords.words('english'))) warnings.filterwarnings('ignore') punctuation = string.punctuationdf=pd.read_csv('gun-violence-data_01-2013_03-2018.csv',parse_dates = ['date']) df.head()
数据集维度
#数据集维度 df.shape #(239677, 29)数据为空的数量
#数据为空的数量 df.isnull().sum()
#增加特征 # Create some additional features df['year'] = df['date'].dt.year df['month'] = df['date'].dt.month df['monthday'] = df['date'].dt.day df['weekday'] = df['date'].dt.weekday df['loss'] = df['n_killed'] + df['n_injured'] try : df.drop(['incident_id', 'incident_url', 'source_url', 'incident_url_fields_missing', 'sources','participant_name'], axis=1, inplace=True) #去掉2013年数据 df=df.loc[df['date'].dt.year!=2013] except : print('process finish') df.head()
df['gun_type_parsed'] = df['gun_type'].fillna('0:Unknown') gt = df.groupby(by=['gun_type_parsed']).agg({'n_killed': 'sum', 'n_injured' : 'sum', 'state' : 'count'}).reset_index().rename(columns={'state':'count'}) results = {} for i, each in gt.iterrows(): wrds = each['gun_type_parsed'].split("||") for wrd in wrds: if "Unknown" in wrd: continue wrd = wrd.replace("::",":").replace("|1","") gtype = wrd.split(":")[1] if gtype not in results: results[gtype] = {'killed' : 0, 'injured' : 0, 'used' : 0} results[gtype]['killed'] += each['n_killed'] results[gtype]['injured'] += each['n_injured'] results[gtype]['used'] += each['count'] gun_names = list(results.keys()) used = [each['used'] for each in list(results.values())] killed = [each['killed'] for each in list(results.values())] injured = [each['injured'] for each in list(results.values())] danger = [] for i, x in enumerate(used): danger.append((killed[i] + injured[i]) / x) trace1 = go.Bar(x=gun_names, y=used, name='SF Zoo', orientation = 'v', marker = dict(color = '#EEE8AA', line = dict(color = '#EEE8AA', width = 1) )) data = [trace1] layout = dict(height=400, title='Which guns have been used?', legend=dict(orientation="h")); fig = go.Figure(data=data, layout=layout) iplot(fig, filename='枪支使用情况统计')
可以发现，手枪在枪击事件中用的最多
#时间格式的转换 df_year=df.groupby(['year'])['n_killed','n_injured'].agg('sum') states_df = df['state'].value_counts() statesdf = pd.DataFrame() statesdf['state'] = states_df.index statesdf['counts'] = states_df.values scl = [[0.0, 'rgb(242,240,247)'],[0.2, 'rgb(218,218,235)'],[0.4, 'rgb(188,189,220)'],\ [0.6, 'rgb(158,154,200)'],[0.8, 'rgb(117,107,177)'],[1.0, 'rgb(84,39,143)']] state_to_code = {'District of Columbia' : 'dc','Mississippi': 'MS', 'Oklahoma': 'OK', 'Delaware': 'DE', 'Minnesota': 'MN', 'Illinois': 'IL', 'Arkansas': 'AR', 'New Mexico': 'NM', 'Indiana': 'IN', 'Maryland': 'MD', 'Louisiana': 'LA', 'Idaho': 'ID', 'Wyoming': 'WY', 'Tennessee': 'TN', 'Arizona': 'AZ', 'Iowa': 'IA', 'Michigan': 'MI', 'Kansas': 'KS', 'Utah': 'UT', 'Virginia': 'VA', 'Oregon': 'OR', 'Connecticut': 'CT', 'Montana': 'MT', 'California': 'CA', 'Massachusetts': 'MA', 'West Virginia': 'WV', 'South Carolina': 'SC', 'New Hampshire': 'NH', 'Wisconsin': 'WI', 'Vermont': 'VT', 'Georgia': 'GA', 'North Dakota': 'ND', 'Pennsylvania': 'PA', 'Florida': 'FL', 'Alaska': 'AK', 'Kentucky': 'KY', 'Hawaii': 'HI', 'Nebraska': 'NE', 'Missouri': 'MO', 'Ohio': 'OH', 'Alabama': 'AL', 'Rhode Island': 'RI', 'South Dakota': 'SD', 'Colorado': 'CO', 'New Jersey': 'NJ', 'Washington': 'WA', 'North Carolina': 'NC', 'New York': 'NY', 'Texas': 'TX', 'Nevada': 'NV', 'Maine': 'ME'} statesdf['state_code'] = statesdf['state'].apply(lambda x : state_to_code[x]) data = [ dict( type='choropleth', colorscale = scl, autocolorscale = False, locations = statesdf['state_code'], z = statesdf['counts'], locationmode = 'USA-states', text = statesdf['state'], marker = dict( line = dict ( color = 'rgb(255,255,255)', width = 2 ) ), colorbar = dict( title = "Gun Violence Incidents") ) ] layout = dict( title = 'State wise number of Gun Violence Incidents', geo = dict( scope='usa', projection=dict( type='albers usa' ), showlakes = True, lakecolor = 'rgb(255, 255, 255)'), ) fig = dict( data=data, layout=layout ) iplot( fig, filename='不同地区枪支暴力事件' )
可以清晰的看到哪个州发生枪击暴力事件的数量最多
df_year.head(1) df_year.plot(figsize=(10,8)) plt.show()
temp = df["state"].value_counts().head(20) #temp.iplot(kind='bar', xTitle = 'State name', yTitle = "No. of incidents", title = 'Top 20 States with highest number of Gun Violence',color="Red")trace1 trace_top20=go.Bar(x=temp.index, y=temp.values) data=[trace_top20] iplot(data,filename='枪支暴力冲突数量最多的前20个州')
此处去掉了13年数据
df['loss'] = df['n_killed'] + df['n_injured'] statdf = df.reset_index().groupby(by=['state']).agg({'loss':'sum', 'year':'count'}).rename(columns={'year':'count'}) statdf['state'] = statdf.index trace1 = go.Bar( x=statdf['state'], y=statdf['count'], name='Count of Incidents', marker=dict(color='rgb(255,10,225)'), opacity=0.6 ) trace2 = go.Bar( x=statdf['state'], y=statdf['loss'], name='Total Loss', marker=dict(color='rgb(58,22,225)'), opacity=0.6 ) data = [trace1, trace2] layout = go.Layout( barmode='group', margin=dict(b=150), legend=dict(dict(x=-.1, y=1.2)), title = 'State wise number of Gun Violence Incidents and Total Loss', ) fig = go.Figure(data=data, layout=layout) iplot(fig, filename='grouped-bar')
先跳过1.6哈~详细的可以看github上的源码哈，地址在文章末尾！
from PIL import Image from wordcloud import WordCloud, STOPWORDS mask = np.array(Image.open('gun2.jpg')) txt = " ".join(df['location_description'].dropna()) wc = WordCloud(mask=mask, max_words=1200, stopwords=STOPWORDS, colormap='copper', background_color='White').generate(txt) plt.figure(figsize=(16,18)) plt.imshow(wc) plt.axis('off') plt.title(''); plt.show()
枪击事件多发在公寓，公园，高校，机场等地
这里先跳过分析的一部分，详细代码看下面github链接哦
df_18=df.ix[df['year']==2018] df_ts=df_18[['n_killed','date']] df_ts.index=df_18['date'] df_ts['n_killed'].plot(figsize=(15,6), color="green") plt.xlabel('Year') plt.ylabel('No. of Deaths') plt.title("Death due to Gun Violence Time-Series Visualization") plt.show()
from fbprophet import Prophet sns.set(font_scale=1) df_date_index = df_18[['date','n_killed']] df_date_index = df_date_index.set_index('date') df_prophet = df_date_index.copy() df_prophet.reset_index(drop=False,inplace=True) df_prophet.columns = ['ds','y'] m = Prophet() m.fit(df_prophet) future = m.make_future_dataframe(periods=365,freq='D') forecast = m.predict(future) fig = m.plot(forecast) plt.show()
打印预测的详细趋势
m.plot_components(forecast); plt.show()
预计2018年的枪击事件数量仍然呈上升趋势，具体在星期五、星期六有较强的上升趋势，应注意防范
1.pip install XXX报错：
parse() got an unexpected keyword argument ‘transport_encoding’
解决：conda install pip
2.No matching distribution found for mpl_toolkits
解决：
下载停用词表出错
Resource ‘corpora/stopwords.zip/stopwords/’ not found. Please
use the NLTK Downloader to obtain the resource: >>>
nltk.download()
Searched in:
- ‘C:\Users\liang/nltk_data’
- ‘C:\nltk_data’
- ‘D:\nltk_data’
- ‘E:\nltk_data’
- ‘j:\Anaconda3\nltk_data’
- ‘j:\Anaconda3\lib\nltk_data’
- ‘C:\Users\liang\AppData\Roaming\nltk_data’
解决方案：
nltk.download(“stopwords”)
成功提示：
[nltk_data] Downloading package stopwords to
[nltk_data] C:\Users\liang\AppData\Roaming\nltk_data…
[nltk_data] Unzipping corpora\stopwords.zip.
Out[7]:
True
去掉DataFrame中某个特征里面符合某条件的数据（如去掉“年份”中为“2013”的数据）
解决方法：
使用df=df.loc[年份=2013]
注：df.iloc[]可以很方便的截取要是用哪些数据
如df.iloc[:, 1:] 表示使用所有行，第一列以后的数据
github源码地址： https://github.com/LIANGQINGYUAN/GunViolence_DataMining
欢迎Star~