数据分析/数据科学Python常用代码块

标签：数据分析 df Python 代码 ## np import my col

换源_python

!pip insall package_name -i https://pypi.douban.com/simple/ 
#从指定镜像下载安装工具包，镜像URL可自行修改

模型服务

keras 部署服务

import tensorflow as tf
## freeze traiing session
def freeze_session(session, keep_var_names=None, output_names=None, clear_devices=True):
    ## get session graph
    graph = session.graph
    with graph.as_default():
        ## remove training related nodes
        freeze_var_names = list(set(v.op.name for v in tf.global_variables()).difference(keep_var_names or []))
        output_names = output_names or []
        output_names += [v.op.name for v in tf.global_variables()]
        input_graph_def = graph.as_graph_def()
        ## remove device info if trained on gpu
        if clear_devices:
            for node in input_graph_def.node:
                node.device = ""
        frozen_graph = tf.graph_util.convert_variables_to_constants(
            session, input_graph_def, output_names, freeze_var_names)
        return frozen_graph
        
from keras import backend as K
frozen_graph = freeze_session(K.get_session(),
                              output_names=[out.op.name for out in model.outputs])
tf.train.write_graph(frozen_graph, "./", "model.pb", as_text=False)

可部署为服务的 Python 脚本

__name__ = 'model'

def func(input_object):
    ## your code here
    return output_object

SQL操作

连接 PostgreSQL

import psycopg2
import pandas as pd

connection = psycopg2.connect(user = "username",
                                  password = "password",
                                  host = "127.0.0.1",
                                  port = "5432",
                                  database = "dn_name")
cursor = connection.cursor()

test_query = """SELECT subject_id, hadm_id, admittime, dischtime, admission_type, diagnosis
FROM admissions
"""

test = pd.read_sql_query(test_query, connection)

单表导出数据

### 从单表中获取数据
### 使用 WHERE 和 HAVING 语句进行过滤
### 用 order_by_list 字段进行降序排序
### conn 为 postgreSQL 的数据库连接对象
query_string = """SELECT [DISTINCT] <select_column_list> [AS <alias_name>]
FROM <table_name>
WHERE <where_condition_1> AND <where_condition_2> OR <where_condition_3>
HAVING <having_condition>
ORDER BY <order_by_list> [DESC]
"""

df = pd.read_sql_query(query_string, conn)

使用 WHERE 语句过滤

### 从单表中获取数据
### 使用 WHERE 语句进行过滤，包含了 IN 包含关系，和使用 LIKE 做模式匹配
### conn 为 postgreSQL 的数据库连接对象
query_string = """SELECT [DISTINCT] <select_column_list> [AS <alias_name>]
FROM <table_name>
WHERE column_name [NOT] IN ( value_1, value_2, ...,value_n)
AND column_name BETWEEN value_1 AND value_2
OR column_name LIKE 'string'
"""

df = pd.read_sql_query(query_string, conn)

使用 HAVING 语句过滤

### 从单表中获取数据
### 使用 HAVING 语句作过滤，包含聚合函数
### conn 为 postgreSQL 的数据库连接对象
query_string = """SELECT [DISTINCT] <select_column_list> [AS <alias_name>]
FROM <table_name>
HAVING [aggregation function] = value_1
AND [aggregation_function] = value_2
"""

df = pd.read_sql_query(query_string, conn)

取出前N条数据

### 从单表中获取数据
### 使用 WHERE 和 HAVING 语句进行过滤
### 用 order_by_list 字段进行降序排序
### 取出头部的N条数据
### conn 为 postgreSQL 的数据库连接对象
query_string = """SELECT [DISTINCT] <select_column_list> [AS <alias_name>]
FROM <table_name>
WHERE <where_condition_1> AND <where_condition_2> OR <where_condition_3>
HAVING <having_condition>
ORDER BY <order_by_list> [DESC]
LIMIT <selected N>
"""

df = pd.read_sql_query(query_string, conn)

多表导出数据

### 从多表中获取数据
### 使用 INNER JOIN 从两表中获取数据
### 使用 WHERE 和 HAVING 语句进行过滤
### conn 为 postgreSQL 的数据库连接对象
query_string = """SELECT [DISTINCT] <select_column_list>
FROM <left_table>
<join_type> JOIN <right_table>
ON <join_condition>  
WHERE <where_condition>
HAVING <having_condition>
ORDER BY <order_by_list> DESC
LIMIT <limit_number>
"""

df = pd.read_sql_query(query_string, conn)

使用聚合函数

### 从多表中获取数据
### 使用 INNER JOIN 从两表中获取数据
### 使用 WHERE 和 HAVING 语句进行过滤
### conn 为 postgreSQL 的数据库连接对象
query_string = """SELECT [aggregation function] (<column_name>)
FROM <left_table>
<join_type> JOIN <right_table>
ON <join_condition>
WHERE <where_condition>
GROUP BY <group_by_all_the_left_columns>
[HAVING <having_condition>]
[ORDER BY <order_by_list>]
[LIMIT <limit_number>]
"""

df = pd.read_sql_query(query_string, conn)

使用 Subquery

### 如果SQL Query过长过复杂，可把Subquery的结果进行调用
### conn 为 postgreSQL 的数据库连接对象

query_string = """--subquery作为一张表
SELECT <column_list> FROM ( 
  SELECT <column_list> FROM table_name
) AS alias_table_name
"""
query_string_1 = """--subquery作为一系列值
SELECT <column_list> FROM <table_name>
WHERE <column_name> IN (
  SELECT <column_name> FROM <table_name> WHERE  <where_condition>
)
"""

df = pd.read_sql_query(query_string, conn)

使用 WITH 语句

### 如果SQL Query过长过复杂，可以将子查询的结果定义为表变量在后期复用
### conn 为 postgreSQL 的数据库连接对象
query_string = """WITH table_variable_1 AS (
  <SELECT query>
),

table_variable_2 AS (
  SELECT * FROM table_variable_1;
)

SELECT * FROM table_variable_2；
"""

df = pd.read_sql_query(query_string, conn)

通用条件表达式 (类似if-else)

### 条件语句
### conn 为 postgreSQL 的数据库连接对象
query_string = """-- 通用条件表达式 (类似if-else)
CASE  
WHEN condition1 THEN result1 -- i.e WHEN count > 5 THEN 1
WHEN condition2 THEN result2 -- i.e. WHEN name = 'elle' THEN 'ELLE'
[...]
[ELSE result_n]
END 
"""

df = pd.read_sql_query(query_string, conn)

查看数据库中所有表名

### 查看数据库中所有表名
### conn 为 postgreSQL 的数据库连接对象
query_string = """
SELECT * FROM pg_tables
WHERE schemaname <> 'pg_catalog' AND schemaname <> 'information_schema'; 
"""

df = pd.read_sql_query(query_string, conn)

窗口函数

### 窗口函数
### conn 为 postgreSQL 的数据库连接对象
query_string = """SELECT <<column_name>,
       window_func() OVER ( [PARTITION BY xx] [ORDER BY xx] )
FROM <table_name>
"""

df = pd.read_sql_query(query_string, conn)

查看表内字段类型

### 查看表内字段类型
### conn 为 postgreSQL 的数据库连接对象
query_string = """
select column_name, data_type 
from information_schema.columns 
where table_name = <table_name>
"""

df = pd.read_sql_query(query_string, conn)

基本操作

装包换源-Python

!pip install package_name -i https://pypi.douban.com/simple/ #从指定镜像下载安装工具包，镜像URL可自行修改

静默安装-Python

!pip install package_name -q

#描述性统计信息

皮尔森相关系数

import numpy as np 

matrix = np.transpose(np.array(X))
np.corrcoef(matrix[0], matrix[1])[0, 1]

## X: array-like
## https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.corrcoef.html

移动平均数

import numpy as np

ret = np.cumsum(np.array(X), dtype=float)
ret[w:] = ret[w:] - ret[:-w]
result = ret[w - 1:] / w

## X: array-like
## window: int

列的分位数

import pandas as pd
## set columns type
my_df['col'] = my_df['col'].astype(np.float64)

## computations for 4 quantiles : quartiles
bins_col = pd.qcut(my_df['col'], 4)
bins_col_label = pd.qcut(my_df['col'], 4).labels

多重聚合(组数据)

## columns settings
grouped_on = 'col_0'  ## ['col_0', 'col_2'] for multiple columns
aggregated_column = 'col_1'

#### Choice of aggregate functions
### On non-NA values in the group
### - numeric choice :: mean, median, sum, std, var, min, max, prod
### - group choice :: first, last, count
## list of functions to compute
agg_funcs = ['mean', 'max']


## compute aggregate values
aggregated_values = my_df.groupby(grouped_on)[aggregated_columns].agg(agg_funcs)

## get the aggregate of group
aggregated_values.ix[group]

以列值排序

import pandas as pd
## my_df 是pandas dataframe
my_df['col_0'].value_counts().sort_index()

用户定义方程(组数据)

## columns settings
grouped_on = ['col_0']
aggregated_columns = ['col_1']

def my_func(my_group_array):
    return my_group_array.min() * my_group_array.count()

### list of functions to compute
agg_funcs = [my_func] ## could be many

## compute aggregate values
aggregated_values = my_df.groupby(grouped_on)[aggregated_columns].agg(agg_funcs)

标准差

import numpy as np
np.std(np.array(X))

## X: array-like
## https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.std.html

在聚合的dataframe上使用用户定义方程

## top n in aggregate dataframe
def top_n(group_df, col, n=2):
    bests = group_df[col].value_counts()[:n]
    return bests

## columns settings
grouped_on = 'col_0'
aggregated_column = 'col'

grouped = my_df.groupby(grouped_on)
groups_top_n = grouped.apply(top_n, aggregated_column, n=3)

所有列(列的数据类型为数值型)

my_df.describe()

以频率降序排序

import pandas as pd
## my_df 是pandas dataframe
my_df['col_0'].value_counts()

一些列的单一属性(如最大值，列的类型为数值型)

my_df["col"].max() ## [["col_0", "col_1"]] 多字段

最大互信息数

import numpy as np

matrix = np.transpose(np.array(X)).astype(float)
mine = MINE(alpha=0.6, c=15, est="mic_approx")
mic_result = []
for i in matrix[1:]:
    mine.compute_score(t_matrix[0], i)
    mic_result.append(mine.mic())
return mic_result

平均数

import numpy as np
np.average(np.array(X))

## X: array-like
## https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.average.html

组数据的基本信息

## columns settings
grouped_on = 'col_0'  ## ['col_0', 'col_1'] for multiple columns
aggregated_column = 'col_1'

#### Choice of aggregate functions
### On non-NA values in the group
### - numeric choice : mean, median, sum, std, var, min, max, prod
### - group choice : first, last, count
### On the group lines
### - size of the group : size
aggregated_values = my_df.groupby(grouped_on)[aggregated_column].mean()
aggregated_values.name = 'mean'

## get the aggregate of group
aggregated_values.ix[group]

行列数

## 获取当前dataframe的形状
nb_rows = my_df.shape[0]
nb_cols = my_df.shape[1]

中位数

import numpy as np
np.median(np.array(X))

## X: array-like
## https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.median.html

数据组的遍历

## columns settings
grouped_on = 'col_0'  ## ['col_0', 'col_1'] for multiple columns

grouped = my_df.groupby(grouped_on)

i = 0
for group_name, group_dataframe in grouped:
    if i > 10:
        break
    i += 1
    print(i, group_name, group_dataframe.mean())  ### mean on all numerical columns

协方差

my_df.cov()

方差

import numpy as np

np.var(np.array(X))

## X: array-like
## https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.var.html

K平均数算法

import numpy as np
from sklearn.cluster import KMeans

k_means = KMeans(k).fit(np.array(X))
result = k_means.labels_
label = result.tolist()
return label, k, k_means.cluster_centers_.tolist(), k_means.inertia_

## k: int, k>=2
## X: array-like
## http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html

所有列的单一属性(如最大值，列的类型为数值型)

my_df.max()

获得类别字段的频数

my_df['category'].value_counts()

查看缺失情况

for col in my_df.columns:
    print("column {} 包含 {} 个缺失值".format(col, my_df[col].isnull().sum()))

查看字段包含unique值的程度

for col in my_df.columns:
    print("column {} 中共有 {} 个unique value".format(col, my_df[col].nunique()))

新建列

使用复杂的方程设值

import numpy as np
def complex_formula(col0_value, col1_value):
    return "%s (%s)" % (col0_value, col1_value)

my_df['new_col'] = np.vectorize(complex_formula)(my_df['col_0'], my_df['col_1'])

标准聚合(使用groupby)

## columns settings
grouped_on = 'col_1'
aggregated_column = 'col_0'

#### Choice of aggregate functions
### On non-NA values in the group
### - numeric choice : mean, median, sum, std, var, min, max, prod
### - group choice : first, last, count
my_df['aggregate_values_on_col'] = my_df.groupby(grouped_on)[aggregated_column].transform(lambda v: v.mean())

使用自定义方程设值

def to_log(v):
    try:
        return log(v)
    except:
        return np.nan
my_df['new_col'] = my_df['col_0'].map(to_log)

用户定义的聚合方程(使用groupby)

def zscore(x):
    return (x - x.mean()) / x.std()
   
my_df['zscore_col'] = my_df.groupby(grouped_on)[aggregated_column].transform(zscore)

缺失值处理

用一个值填补多列的缺失值

my_df[['col_0', 'col_1']] = my_df[['col_0', 'col_1']].fillna(value)

用一个值填补一列的缺失值

my_df['col'] = my_df['col'].fillna(value)

去除在指定列中带有缺失值的数据条目

cols = ['col_0', 'col_1']
records_without_nas_in_cols = my_df.dropna(subset=cols)

用最后一个或缺失值的下一个数据填补缺失值

## - ffill : propagate last valid observation forward to next valid
## - backfill : use NEXT valid observation to fill gap
my_df['col'] = my_df['col'].fillna(method='ffill')

用一个由聚合得出的值填补缺失值

grouped_on = 'col_0' ## ['col_1', 'col_1'] ## for multiple columns

#### Choice of aggregate functions
### On non-NA values in the group
### - numeric choice : mean, median, sum, std, var, min, max, prod
### - group choice : first, last, count
def filling_function(v):
    return v.fillna(v.mean())
                    
my_df['col'] = my_df.groupby(grouped_on)['col'].transform(filling_function)

以列为单位审查数据集

my_df.info()

去除所有任何缺失值的数据条目

records_without_nas = my_df.dropna()

Pandas基本操作

计算频率来转置表

freqs = my_df.pivot_table(
    rows=["make"],
    cols=["fuel_type", "aspiration"],
    margins=True     ## add subtotals on rows and cols
)

水平连接数据

import pandas as pd
two_dfs_hconcat = pd.concat([my_df, my_df2], axis=1)

将dataframe的index重置成标准值

my_df_no_index = my_df.reset_index()
## my_df_no_index.index is now [0, 1, 2, ...]

以元组(tuple)形式按行写入数据

import pandas as pd
py_recipe_output = pd.read_csv("data.csv")
writer = py_recipe_output.get_writer()

## t is of the form :
##   (value0, value1, ...)

for t in data_to_write:
    writer.write_tuple(t)

数据重构

#Pivot
df3 = df2.pivot( index = 'Date', 
columns = 'Type', values = 'Value') #行变列

#Pivot Table
df4 = pd.pivot_table( df2,
values='Value',
index = 'Date', 
columns='Type'] #行变列

#Stack/Unstack
stacked = df5.stack( ) 
stacked.unstacked( )

#Melt
pd.melt(df2, id_vars=["Date"], 
value_vars=["Type","Value"], 
value_name="Observations") 
#将列变行

用数据的位置获取数据

my_record = my_df.iloc[1]  ## !! get the second records, positions start at 0

从列新建虚数据

import pandas as pd
dummified_cols = pd.get_dummies(my_df['col']
    ## dummy_na=True ## to include NaN values
    )

合并有相同名称的列的两个dataframe

import pandas as pd
## my_df 是一个pandas dataframe
merged = my_df.merge(my_df2,
on='col', ## ['col_0', 'col_1'] for many
how="inner",
## suffixes=("_from_my_df", "_from_my_df2"))

合并两个dataframe没有相同名称的列

## 每次只合并两个dataframe
## 合并方式 : 'left', 'right', 'outer', 'inner'
import pandas as pd
## my_df 是一个pandas dataframe
merged = my_df.merge(my_df2,
left_on=['col_0', 'col_1'],
right_on=['col_2', 'col_3'],
## suffixes=("_from_my_df", "_from_my_df2")
how="inner")

分组数据

#aggregation
df2.groupby(
by=['Data', 'Type']).mean()
df4.groupby(level=0).sum()
df4.groupby(level=0).agg
({'a':lama x:sum(x)/len(x), 
'b':np.sum})

#Transformation
customSum=lamda x:(x+x%2)
df4.groupby
(level=0).transform(customSum)

使用目标值的位置定位替换值

import pandas as pd
## my_df 是一个pandas dataframe
my_df[]'col'].iloc[0] = new_value  ## replacement for first record

分簇

import pandas as pd
## Set columns type
my_df['col'] = my_df['col'].astype(np.float64)

## Computations
bins = [0, 100, 1000, 10000, 100000] ## 5 binned, labeled 0,1,2,3,4
bins_col = pd.cut(my_df['col'], bins)
bins_col_label = pd.cut(my_df['col'], bins).labels

获取元组(tuple)的迭代器

import pandas as pd
my_dataset = pd.read_csv("path_to_dataset")

i = 0
for my_row_as_tuple in my_dataset.iter_tuples():
    if i > 10:
        break
    i += 1
    print (my_row_as_tuple)

获取文档

## Everywhere
print(my_df.__doc__)
print(my_df.sort.__doc__)

## When using notebook : append a '?' to get help
my_df?
my_df.sort?

复制数据

s3.unique() 
#返回唯一值
df2.duplicated('Tyepe') 
#检查重复值
df2.drop_duplicates(
'Type', keep=last'') 
#丢弃重复值
df.index.duplicated() 
#检查索引重复

重命名所有列

my_df.columns = ['new_col_0', 'new_col_1']  ## needs the right number of columns

设置聚合方程进行交叉制表

## 交叉制表是统计学中一个将分类数据聚合成列联表的操作
import pandas as pd
## my_df 是一个pandas dataframe
stats =  pd.crosstab(
    rows=my_df["make"],
    cols=[my_df["fuel_type"], my_df["aspiration"]],
    values=my_df["horsepower"],
    aggfunc='max',   ## aggregation function
    margins=True     ## add subtotals on rows and cols
)

高级索引

#basic
df3.loc[:, (df3>1).any()] 
#选列 其中任意元素>1
df3.loc[:, (df3>1).all()] 
#选列 其中所有>1
df3.loc[:, df3.isnull().any()] 
#选列 其中含空
df3.loc[:,df3.notnull().all()] 
#选列 其中不含空

df[(df.Country.isin(df2.Type))] 
#寻找相同元素
df3.filter(items=["a","b"]) 
#根据值筛选
df.select(lamda x: not x%5) 
#选特定元素

s.where(s>0) 
#数据分子集

df6.query('second >first') 
#query 数据结构


#selecting
df.set_index('Country') 
#设置索引
df4 = df.reset_index() 
#重置索引
df =df.rename(index = str, 
columns={"Country":"cntry", 
"Capital":"cptl","Population":"ppltn"}) 
#重命名数据结构

#Reindexing
s2 = s.reindex(['a', 'c', 'd', 'e', 'b'])
#Forward Filling
df.reindex(range(4), 
method='ffill')
#Backward Filling
s3 = s.reindex(range(5), 
method='bfill')

#MultiIndexing
arrays = [np.array([1,2,3]), 
np.array([5,4,3])]
df5 = pd.DataFrame(
np.random.rand(3,2), index = arrays)
tuples = list(zip( *arrays))
index = pd.MultiIndex.from_tuples(
tuples, names=['first', 'second'])
df6 = pd.DataFrame(
np.random.rand(3,2), index = index)
df2.set_index(["Data","Type"])

使用某种法则替换列中的值

import numpy as np
rules = {
    value: value1,
    value2: value3,
    'Invalid': np.nan  ## replace by an true invalid value
}

my_df['col'] = my_df['col'].map(rules)

遍历数据字典

import pandas as pd
my_dataset = pd.read_csv("path_to_dataset")

i = 0
for my_row_as_dict in my_dataset.iter_rows():
    if i > 10:
        break
    i += 1
    print my_row_as_dict

垂直连接数据

two_dfs_vconcat = my_df1.append(my_df2)

计算频率进行交叉制表

## 交叉制表是统计学中一个将分类数据聚合成列联表的操作
import pandas as pd
## my_df 是一个pandas dataframe
freqs = pd.crosstab(
    rows=my_df["make"],
    cols=[my_df["fuel_type"], my_df["aspiration"]]
)

迭代

df.iteritems( ) #列索引 
df.iterrows( ) #行索引

结合数据

#Merge
pd.merge(data1, data2, 
how='left', on='X1')
pd.merge(data1, data2,
how='right', on='X1')
pd.merge(data1, data2, 
how='inner', on='X1')
pd.merge(data1, data2, 
how='outer', on='X1')

#join
data1.join(data2, how='right')

#concatenate
#vertical
s.append(s2)
#horizontal/vertical
pd.concat([s,s2], axis=1,
 keys=['One','Two'])
pd.concat([data1, data2], 
axis=1, join='inner')

用index获取数据

my_record = my_df.loc[label] ## !! If the label in the index defines a unique record

使用目标值的label定位替换值

import pandas as pd
## my_df 是一个pandas dataframe
my_df['col'].loc[my_label] = new_value

获取多列数据

cols_names = ['col_0', 'col_1']
my_cols_df = my_df[cols_names]

使用函数新增列

def remove_minus_sign(v):
    return str.replace('-', ' ', max=2)

my_df['col'] = my_df['col'].map(remove_minus_sign)

基本的数据排序

my_df = my_df.sort('col', ascending=False)

使用条件定位替换值

import pandas as pd
## my_df_orig 是一个pandas dataframe
cond = (my_df_orig['col'] != value)
my_df_orig['col'][cond] = "other_value"

从dataframe随机抽取N行数据

import random
n = 10
sample_rows_index = random.sample(range(len(my_df)), 10)
my_sample = my_df.take(rows)
my_sample_complementary = my_df.drop(rows)

按列里的值过滤或去除数据

cond = (my_df['col'] == value)

## multiple values
## cond = my_df['col'].isin([value1, value2])

## null value
## cond = my_df['col'].isnull()

## exclude (negate condition)
## cond = ~cond

my_records = my_df[cond]

对多列数据排序

my_df = my_df.sort(['col_0', 'col_1'], ascending=[True, False])

将dataframe的index设为列名

my_df_with_col_index = my_df.set_index("col")
## my_df_col_index.index is [record0_col_val, record1_col_val, ...]

将数据集加载成多个dataframe

import pandas as pd
my_dataset = pd.read_csv("data.csv")

for partial_dataframe in my_dataset.iter_dataframes(chunksize=2000):
    ## Insert here applicative logic on each partial dataframe.
    pass

以字典(dict)形式按行写入数据

import pandas as pd
py_recipe_output = pd.read_csv("data.csv")
writer = py_recipe_output.get_writer()

## d is of the form :
##   {'col_0': value0, 'col_1': value1, ...}

for d in data_to_write:
    writer.write_row_dict(r)

使用dataframe的index新建列

my_df['new_col'] = my_df.index

可视化

import Matplotlib.pyplot as plt
s.plot()
plt.show()

df2.plot()
plt.show()

设置聚合方程转置表

stats = my_df.pivot_table(
    rows=["make"],
    cols=["fuel_type", "aspiration"],
    values=["horsepower"],
    aggfunc='max',   ## aggregation function
    margins=True     ## add subtotals on rows and cols
)

使用boolean formula过滤或去除数据

import pandas as pd
## single value
cond1 = (my_df['col_0'] == value)
cond2 = (my_df['col_1'].isin([value1, value2]))
## boolean operators :
## - negation : ~  (tilde)
## - or : |
## - and : &
cond = (cond1 | cond2)
my_records = my_df[cond]

日期

df2['Date'] = pd.to_datetime
(df2['Data'])
df2['Date'] = pd.date_range
('2000-1-1', periods=6, freq ='M')
dates = [datetime(2012,5,1), 
datetime(2012,5,2)]
index = pd.DatetimeIndex(dates)
index = pd.date_range
(datetime(2012,2,1)), end, freq='BM'

重命名指定列

my_df = my_df.rename(columns = {'col_1':'new_name_1', 'col_2':'new_name_2'})

Numpy基本操作

array处理

import numpy as np

#Transposing Array
I = np.transpose(b) #转置矩阵
i.T #转置矩阵

#Changing Array Shape
b.ravel() #降为一维数组
g.reshape(3,-2) #重组

#Adding/Removing Elements
h.resize((2,6)) #返回shape(2,6)
np.append(h,g) #添加
np.insert(a,1,5) #插入
np.delete(a,[1]) #删除

#Combining Arrays
np.concatenate((a,d), axis=0) #连结
np.vstack((a,b)) #垂直堆叠
np.r_[e,f] #垂直堆叠
np.hstack((e,f)) #水平堆叠
np.column_stack((a,d)) #创建水平堆叠
np.c_[a,d] ##创建水平堆叠

#splitting arrays
np.hsplit(a,3) #水平分离
np.vsplit(c,2) #垂直分离

创建array

import numpy as np
a = np.array([1,2,3])   #创建数组
b = np.array([(1.5,2,3), (4,5,6)], 
dtype=float)
c = np.array([(1.5,2,3), (4,5,6)], 
[(3,2,1), (4,5,6) ] ], dtype=float)

np.zeros((3,4))  #创建0数组
np.ones((2,3,4), dtype=np.int16)  #创建1数组
d = np.arrange(10,25,5)  #创建相同步数数组
np.linspace(0,2,9)  #创建等差数组

e = np.full((2,2), 7) #创建常数数组
f = np.eye(2) #创建2x2矩阵
np.random.random((2,2)) #创建随机数组
np.empty((3,2)) #创建空数组

排序array

#对数组进行排序
a.sort()
c.sort(axis=0)

索引

import numpy as np
#subsetting
a[2] #选取数组第三个元素
b[1,2] #选取2行3列元素

#slicing
a[0:2] #选1到3元素
b[0:2,1] #选1到2行的2列元素
b[:1] #选所有1行的元素
c[1,...] #c[1,:,:]
a[ : :-1]  #反转数组

#Boolean Indexing
a[a<2] #选取数组中元素<2的

#Fancy Indexing
b[[1,0,1,0], [0,1,2,0]]
#选取[1,0],[0,1],[1,2],[0,0]
b[[1,0,1,0][:, [0,1,2,0]]] 
#选取矩阵的一部分

基本运算

import numpy as np

#arithmetic operation算术运算
g = a - b
np.subtract(a,b) #减法
b+a
np.add(b,a) #加法
a / b
np.divide(a,b) #除法
a * b
np.multiply(a,b) #乘法
np.exp(b) #指数
np.sqrt(b) #开方
np.sin(a) #sin函数
np.cos(b) #cos函数
np.log(a) #log函数
e.dot(f) #内积

#Comparison比较
a == b #元素
a < 2 #元素
np.array_equal(a,b) #数组 

#Aggregate Functions 函数
a.sum() #求和
b.min() #最小值
b.max(axis=0) #最大值数组列
b.cumsum(axis=1) #元素累加和
a.mean() #平均值
b.median() #中位数
a.corrcoef() #相关系数
np.std(b) #标准差

检查array

a.shape #数组维度
len(a) #数组长度
b.ndim #数组维度数量
e.size #数组元素数量
b.dtype #元素数据类型
b.dtype.name #数据类型名
b.astype(int) #改变数组类型

#asking for help更多信息
np.info(np.ndarray.dtype)

输出array

import numpy as np
print(my_array) #打印数组

#saving &Loading on disk保存到磁盘
np.save('my_array', a) 
np.savez('array.npz', a, b)
np.load('my_array.npy')

#saving &Loading Text files保存到文件
np.loadtxt("my file.txt")
np.genfromtxt("my_file.csv", delimiter=',')
np.savetxt("marry.txt", a, delimiter=" ")

数据类型

np.int64 #64位整数
np.float32 #标准双精度浮点
np.complex #复杂树已浮点128为代表
np.bool #true&false
np.object #python object
np.string_ #固定长度字符串
np.unicode_ #固定长度统一码

复制array

#复制数组
h = a.view() 
np.copy(a)
h = a.copy()

Matplotlib基本操作

数据录入

import numpy as np
#1D data
x = np.linspace(0, 10, 100)
y = np.cos(x)
z = np.sin(x)

#2D data or images
data = 2* np.random.random((10,10))
data2 = 3 * np.random.random((10,10))
Y, X = np.mgrid[-3:3:100j, -3:3:100j]
U = -1-X**2+Y
V = 1+X-Y**2
from Matplotlib.cbook 
import get_sample_data
img = np.load(get_sample_data
('axes_grid/bivariate_normal.npy'))

显示与保存

#save plot
plt.savefig('foo.png')
plt.savefig('foo.png', transparent=True)

#show plot
plt.show()

#close &clear
plt.cla() #clear an axis
plt.clf() #clear the entire figure
plt.close() #close a window

图像设置

import matplotli.pyplot as plt

#figure
fig = plt.figure( )
fig2 = plt.figure( figsize = 
plt.figaspect(2,0) )

#axes
fig.add_axes( )
ax1 = fig.add_subplot( 221 ) 
#row-col-num
ax3 = fig.add_subplot( 212 )
fig3, axes = plt.subplots(
nrows=2, ncols=2)
fig4, axes2 = plt.subplots( ncols=3 )

自定义

#colors
plt.plot(x, x, x, x**2, x, x**3)
ax.plot(x, y, alpha = 0.4)
ax.plot(x, y, c='k')
fig.colorbar(im, orientation='horizontal')
im = ax.imshow(img, cmap='seismic')

#markers
fig, ax = plt.subplots()
ax.scatter(x,y,marker='.')
ax.plot(x,y,marker='o')

#linestyles
plt.plot(x,y,linewidth=4.0)
plt.plot(x,y,ls='solid')
plt.plot(x,y,ls='--')
plt.plot(x,y,'--',x**2,y**2,'-.')
plt.setp(lines, color='r',linewidth=4.0)

#Text & Annotations
ax.text(1,-2.1, 
'Example Graph', style='italic')
ax.annotate("Sine", xy=(8,0),
xycoords='data',xytext=(10.5,0), 
textcoords='data', arrowprops=
dict(arrowstyle="->",connectionstyle="arc3"),)

#mathtext
plt.title(r'$sigma_i=15$',
 fontsize=20)

#Limits,Legends&Layouts
#Limites&Autoscaling
ax.margins(x=0.0, y=0.1) 
#Add padding to a plot
ax.axis('equal') 
#Set the aspect ratio of the plot to 1
ax.set(xlim=[0,10.5],ylim=[-1.5,1.5]) 
#Set limits for x- and y-axis
ax.set_xlim(0,10.5) 
#Set limits for x-axis

#Legends
ax.set(title='An Example Axes',
ylabel='Y-Axis',xlabel='X-Axis') 
#Set a title and x- and y-axis labels
ax.legend(loc='best') 
#No overlapping plot elements

#Ticks
ax.xaxis.set(ticks=range(1,5),
ticklabels=[3,100,-12,"foo"]) 
#Manually set x-ticks
ax.tick_params(axis='y',
direction ='inout',length=10) 
#Make y-ticks longer and go in and out

#Subplot Spacing
fig3.subplots_adjust(wspace=0.5, 
hspace=0.3,left=0.125,right=0.9,
top=0.9,bottom=0.1) 
#Adjust the spacing between subplots
fig.tight_layout() 
#Fit subplots in to the figure area

#Axis Spines
ax1.spines['top'].set_visible(False) 
#Make the top axis line for a plot invisible
ax1.spines['bottom'].set_position(('outward',10)) 
#Move the bottom axis line outward

图像绘制

#1D data
fig, ax = plt.subplots()
lines = ax.plot(x,y) 
#Draw points with lines or makers connecting them
ax.scatter(x,y)
#Draw unconnected points, scaled or colored
axes[0,0].bar([1,2,3],[3,4,5]) 
#Plot vertical rectangles
axes[1,0].barh([0.5,1,2.5], [0,1,2]) 
#Plot horizontal rectangles
axes[1,1].axhline(0.45) 
#Draw a horizontal line across axes
axes[0,1].axvline(0.65) 
#Draw a vertical line across axes
ax.fill(x,y,color='blue') 
#Draw filled polygons
ax.fill_between(x,y,color='yellow') 
#Fill between y-values and o

#2D data or images
fig, ax = plt.subplots( ) 
#Colormapped or RGB arrays
im = ax. imshow(img, cmap='gist_earth', 
interpolation='nearest',vmin=-2,vmax=2)
axes2[0].pcolor(data2) 
#Pseudocolor plot of 2D array
axes2[0].pcolormesh(data) 
#Pseudocolor plot of 2D array
CS = plt.contour(Y,X,U)
#Plot contours
axes2[2].contourf(data1) 
#Plot filled contours
axes2[2] = ax.clabel(CS)
#Label a contour plot

#Vector Field
axes[0,1].arrow(0,0,0.5,0.5) 
#Add an arrow to the axes
axes[1,1].quiver(y,z) 
#Plot a 2D field of arrows
axes[0,1].streamplot(X,Y,U,V) 
#plot a 2D field of arrows

#Data Distributions
ax1.hist(y) #Plot a histogram
ax3.boxplot(y) 
#make a box and whisker plot
ax3.violinplot(z) 
#make a violin plot

以pandas dataframe制图

所有列(分隔)

my_df.plot(kind='line', subplots=True, figsize=(8,8))

所有列(交叉)

my_df.plot(kind='line', alpha=0.3)

散点图

my_df.plot(kind='scatter', x='col1', y='col2',
    ## c='col_for_color', s=my_df['col_for_size']*10
    );

单一列

my_df['col_name'].plot(kind='hist', bins=100)

Seaborn基本操作

数据导入

import pandas as pd
import numpy as np
uniform_data = np.random.rand(10,12)
data = pd.DataFrame( ( 'x': np.array(1, 101), 
'y': np.random.normal( 0,4,100 ) ) )

#内部数据集
titanic = sns.load_dataset( "titanic" ) 
iris = sns.load_dataset( "iris" )

图像绘制

#axis grids
g = sns.FacetGrid(titanic, 
col = "survived", row = "sex" ) 
#subplot grid
g = g.map( pot.hist, "age")

sns.factorplot( x = "pclass",
y = "survived", hue = "sex", 
data = titanic) ## categorical plot
sns. lmplot( x = "sepal_width",
 y = "sepal_length", hue = "species", 
data = iris) 
#plot data and regression model

h = sns.PairGrid( iris ) 
#subplot grid for plotting pairwise relationships
h = h.map( pot.scatter) 
sns.pairplot( iris ) 
#plot pairwise bivariate distributions
i = sns.JointGrid( x = "x",
 y = "y", data= data )
 #grid for bivariate plot with marginal
i = i.plot( sns.regplot, sns.distplot )
sns.jointplot( "sepal_length", 
"sepal_width", data = iris, kind = 'kde')
#plot bivariate distribution


#Categorical Plots

#Scatterplot
sns.stripplot( x = "species", 
y = "petal_length", data = iris)
sns.swarmplot( x = "species", 
y = "petal_length", data = iris)

#bar chart
sns.barplot( x = "sex", y = "survived",
 hue = "class", data = titanic)

#count plot
sns.countplot( x = "deck", 
data = titanic, palette = "Greens_d")

#Point Plot
sns.pointplot( x = "class", y = "survived",
hue = "sex", data = titanic, 
palette ={ "male": "g", "female": "m"}, 
markers = [ "^","o"], linestyles = ["-","--"])

#Boxplot
sns.boxplot( x = "alive", y = "age",
hue = "adult_male", data = titanic)
sns.boxplot( data = iris, orient = "h")

#Violinplot
sns.violinplot( x = "age", y = "sex",
 hue = "survived", data = titanic ) 


#regression plots
sns.regplot(x="sepal_width", 
y ="sepal_length", data=iris, ax=ax)

#distribution plots
plot = sns.displot(data.y, 
kde=False, color="b")

#Matrix plots
sns.heatmap(uniform_data, 
vmin=0, vmax=1)

自定义

#Axisgrid Objects
g.despine( left = True) 
#remove left spine
g.set_ylabels( "Survived") 
#set the labels of the y-axis
g.set_xticklabels( rotation = 45) 
#set the tick labels for x
g.set_axis_labels( "Survived", "Sex")
 #set the axis labels
h.set( xlim = (0,5), ylim = (0,5), 
xticks = [0,2.5,5], yticks =[0,2.5,5] )
#set the limit and ticks of the x- and y-axis

#Plot
plt.title( "A title" ) 
#add plot title
plt.ylabel( "Survived" ) 
#adjust the label of the y-axis
plt.xlabel( "Sex" ) 
#adjust the label of the x-axis
plt.ylim(0,100)
 #adjust the limits of the y-axis
plt.xlim(0,10) 
#adjust the limits of the x-axis
pot.setp(ax, yticks =[0,5]) 
#adjust a plot property
plt.tight_layout() 
#adjust subplot params

显示与保存

plt.show( ) #显示图像
plt.saveflg( "foo.png" ) #存储
plt.saveflg( "foo.png", transparent = True) 
#存储透明图像

#关闭与清除
plt.cla( ) #清除axis
plt.clf( ) #清除整个图像
plt.close( ) #关闭窗口

图像设置

#创建图像绘制窗口
f, ax = ply.subplots( figsize = ( 5,6 ) )

#seaborn style
sis.set( ) #重置
sns.set_style( "whitegrid" )
#设置matplotlib parameters
sns.set_style( "ticks",
( "xtick.major.size": 8, 
"ytick.major.size": 8 ) )
sns.axes_style( "whitegrid" ) 
#返回一个参数指引

#context functions
sns.set_context( "talk" ) 
#设置内容为“talk”
sns.set_context( "notebook",
 font_scale = 1.5, 
rc = ( "lines.linewidth": 2.5 ) ) 
#设置内容为“notebook”

#color palette
sns.set_palette( "husl",3 )
 #定义color palette
sns.color_palette( "husl" )
flatui = ( “#9b59b6”,"#3498db",
"#95a5a6","#e74c3c",
"#34495e","#2ecc71")
sns.set_palette( flatui )
#定义自己的color palette

Bokeh基本操作

输出

#Notebook: 
from bokeh.io 
import output_notebook, show
output_notebook( )

#HTML:

#Standalone HTML 
from bokeh.embed 
import file_html; 
from bokeh.resources 
import CDN; 
html=file_html(p,CDN,"my_plot");
from bokeh.io 
import output_file,show; 
output_file('my_bar_chart.html',
mode='cdn')

#Components:
from bokeh.embed 
import components; 
script, div=components(p)

#PNG: 
from bokeh.io import export_png;
export_png(p,filename="plot.png")

#SVG:
from bokeh.io import export_svgs;
p.output_backend="svg";
export_svgs(p,filename="plot.svg")

数据导入

import numpy as np
import pandas as pd
df=pd.DataFrame(
np.array([[ 33.9, 4, 65, 'US'], 
[32.4,4, 66, 'Asia'], 
[21.4, 4,109, 'Europe'] ] ), 
columns = [ 'mpg', 'cyl', 'hp', 'origin'],
index= [ 'Toyota', 'Fiat', 'Volvo'] )

from bokeh.models
import ColumnDataSource 
cds_df=ColumDataSource(df)

图像绘制

from bokeh.plotting import figure 
p1 = figure( plot_width = 300, 
tools = 'pan, box_zoom' )  
p2 = figure( plot_width = 300, 
plot_height = 300, 
x_range = (0,8), y_range = (0,8) )
p3 = figure( )

可视化

#Glyphs

#Scatter markers 
p1.circle( np.array( [1,2,3]), 
np.array([3,2,1]), fill_color = 'white')
p2.square(np.array([1.5,3.5,5.5]), 
[1,4,3], color = 'blue', size = 1)

#Line Glyphs: 
p1.line( np.array( [1,2,3,4], 
[3,4,5,6], ), line_width = 2)
p2.multi_line(pd.DataFrame(),
pd.DataFrame(),color='blue')



#Customized Glyphs

#Selection and Non-Selection Glyphs
p = figure( tools = 'box_select')
p.circle( 'mpg', 'cyl', source = cds_df, 
selection_color = 'red', 
nonselection_alpha = 0.1)

#Hover Glyphs: 
from bokeh.models import HoverTool 
hover = HoverTool( tooltips = None, 
mode = 'vline' )

#Colormapping:
from bokeh.models 
import CategoricalColorMapper
color_mapper = CategoricalColorMapper
( factors = ['US', 'Asia', 'Europe'], 
palette = ['blue', 'red', 'green'])
p3.circle( 'mpg', 'cyl', source = cds_df, 
color = dict( field = 'orign',
transform = color_mapper), legend = 'Orign')



#Legend Location

#Inside Plot Area: 
p.legend.location = 'bottom_left'

#Outside Plot Area: 
from bokeh.models import Legend
r1 = p2.asterisk( 
np.array( [1,2,3]), np.array([3,2,1]))
r2 = p2.line([1,2,3,4], [3,4,5,6])
Legend = Legend( items = [("One",
 [p1,r1]]), ("Two", [r2])], location = (0,-30))
p.add_layout( legend,'right')

#Legend Orientation
p.legend.orientation = "horizontal"
p.legend.orientation = "vertical"

#Legend Background &Border
p.legend.border_line_color = "navy"
p.legend.background_fill_color = "white"



#Rows & Columns Layout

#Rows:
from bokeh.layouts import row
layout = row( p1,p2,p3)

#Columns 
from bokeh.layouts import columns
layout = row( column ( p1,p2), p3)

#Nesting Rows&Columns 
layout = row( column (p1,p2), p3)

#Grid Layout:
from bokeh.layouts import gridplot
row1 = [p1, p2]
row2 = [p3]
layout = gridplot ( [ [p1,p2], [p3]])

#Tabbed Layout
from bokeh.models.widgets import Panel, Tabs
tab1 = Panel ( child = p1, title = "tab1" )
tab2 = Panel ( child = p1, title = "tab2" )
layout = Tabs( tabs = [tab1,tab2])



#Linked Plots

#linked Axes
p2.x_range = p1.x_range
p2.y_range = p1.y_range

#Linked Brushing
p4 = figure( plot_width = 100, 
tools = 'box_select, lasso_select')
p4.circle( 'mpg', 'cyl', source = cds_df)
p5 = figure( plot_width = 200, 
tools = 'box_select, lasso_select')
p5.circle( 'mpg', 'hp', source = cds_df)
layout = row( p4, p5)

显示与保存

show(p1)
save(p1)
show(layout)
save(layout)

seaborn可视化

swarmplot

import seaborn as sns
%matplotlib inline
sns.swarmplot(x='species',y='petal_length',data=my_data) ## 以iris数据集为例

pairplot

import seaborn as sns
sns.pairplot(iris,hue='Species')

countplot

import seaborn as sns;titanic= sns.load_dataset('titanic')
sns.countplot(y="deck", hue="class", data=titanic, palette="Greens_d") ## 以titanic数据集为例

violinplot

import seaborn as sns;iris = sns.load_dataset('iris')
sns.violinplot(x='species',y='sepal_length',data=iris) ## 以iris数据集为例

pointplot

import seaborn as sns; iris = sns.load_dataset('iris')
sns.pointplot(x='species',y='petal_width',data=iris) ## 以iris数据集为例

boxplot

import seaborn as sns
sns.boxplot(x='species',y='Petal.Length',data=iris) ## 以iris数据集为例

stripplot

import seaborn as sns; iris = sns.load_dataset('iris')
sns.stripplot(x='species',y='sepal_length',data=iris)

jointplot

import seaborn as sns; sns.set(style="ticks", color_codes=True);tips = sns.load_dataset("tips")
g = sns.JointGrid(x="total_bill", y="tip", data=tips) ## 以tips数据集为例
g = sns.JointGrid(x="total_bill", y="tip", data=tips)
g.plot(sns.regplot, sns.distplot)

heatmap

import numpy as np; np.random.seed(0)
import seaborn as sns; sns.set()
uniform_data = np.random.rand(10, 12)
sns.heatmap(uniform_data)

tsplot

import numpy as np; np.random.seed(22)
import seaborn as sns; sns.set(color_codes=True)
x = np.linspace(0, 15, 31);data = np.sin(x) + np.random.rand(10, 31) + np.random.randn(10, 1)
sns.tsplot(data=data)

distplot

import seaborn as sns, numpy as np,matplotlib.pyplot as plt
%matplotlib inline
sns.set(); np.random.seed(0)
x = np.random.randn(100)
sns.distplot(x)

kdeplot

import numpy as np; np.random.seed(10)
import seaborn as sns; sns.set(color_codes=True)
mean, cov = [0, 2], [(1, .5), (.5, 1)]
x, y = np.random.multivariate_normal(mean, cov, size=50).T
sns.kdeplot(x,y)

barplot

import seaborn as sns; titanic=sns.load_dataset('titanic')
sns.barplot(x="sex", y="survived", hue="class", data=titanic)

数据清洗

去除所有重复数据

import pandas as pd
## my_df 是一个pandas dataframe
unique_records = my_df.drop_duplicates()

归一化

import numpy as np
from sklearn import preprocessing

raw_result = preprocessing.Normalizer().fit_transform(np.array(X))
result = raw_result.tolist()

## X: array-like
## http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.Normalizer.html

多项式数据变换

import numpy as np
from sklearn import preprocessing

raw_result = preprocessing.PolynomialFeatures().fit_transform(np.array(X))
result = raw_result.tolist()

## X: array-like
## http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.PolynomialFeatures.html

去除特定列中的重复数据

import pandas as pd
## my_df 是一个pandas dataframe
unique_records_for_cols = my_df.drop_duplicates(cols=['col_1', 'col_0'])

二值化

import numpy as np
from sklearn import preprocessing

raw_result = preprocessing.Binarizer(threshold=t).fit_transform(np.array(X))
result = raw_result.tolist()

## X: array-like
## t: float
## http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.Binarizer.html

类别转数字

## Label1 和 Label2 表示 类别
target = pd.Series(map(lambda x: dict(Label1=1, Label2=0)[x], my_df.target_col.tolist()), my_df.index)
my_df.target_col = target

特征缩放

from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)

无纲量化

正态分布

import numpy as np
from sklearn import preprocessing

raw_result = preprocessing.StandardScaler().fit_transform(np.array(X))
result = raw_result.tolist()

## X: array-like
## http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html

伯努利分布

import numpy as np
from sklearn import preprocessing

raw_result = preprocessing.MinMaxScaler().fit_transform(np.array(X))
result = raw_result.tolist()

## X: array-like
## http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MinMaxScaler.html

基础维度变化

用拆堆栈进行变化(long format -> wide format)

pivoted = base.unstack(level=1)

用栈变化(wide format -> long format)

stacked = pivoted.stack(dropna=True)

使用melt降维(wide format -> long format)

## melt是一种降维的方法
import pandas as pd
rows = pd.melt(pivoted, id_vars=["type"])

转置

pivoted = rows.pivot(index="type", columns="make", values="qty")

特征选取

递归特征消除法

import numpy as np
from sklearn import feature_selection
from sklearn.linear_model import LogisticRegression

matrix = np.array(X)
target = np.array(target)
temp = feature_selection.RFE(estimator=LogisticRegression(), n_features_to_select=n_features).fit(matrix, target)
scores = temp.ranking_.tolist()
indx = temp.support_.tolist()
result = temp.transform(matrix).tolist()
return scores, indx, result

## X: array-like
## target: array-like
## n-features: int
## http://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.RFE.html

方差选择

import numpy as np
from sklearn import feature_selection

matrix = np.array(X)
temp = feature_selection.VarianceThreshold(threshold=t).fit(matrix)
scores = [np.var(el) for el in matrix.T]
indx = temp.get_support().tolist()
result = temp.transform(matrix).tolist()
return scores, indx, result

## X: array-like
## t: float
## http://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.VarianceThreshold.html

方差选择

import numpy as np
from sklearn import feature_selection

matrix = np.array(X)
temp = feature_selection.VarianceThreshold(threshold=t).fit(matrix)
scores = [np.var(el) for el in matrix.T]
indx = temp.get_support().tolist()
result = temp.transform(matrix).tolist()
return scores, indx, result

## X: array-like
## t: float
## http://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.VarianceThreshold.html

互信息选择法

from minepy import MINE
import numpy as np
from sklearn import feature_selection

matrix = np.array(X)
target = np.array(target)
def mic(x, y):
    m = MINE()
    m.compute_score(x, y)
    return (m.mic(), 0.5)
temp = feature_selection.SelectKBest(lambda X, Y: np.array(list(map(lambda x: mic(x, Y), X.T))).T[0], k=k).fit(matrix, target)
scores = temp.scores_.tolist()
indx = temp.get_support().tolist()
result = temp.transform(matrix).tolist()
return scores, indx, result

## X: array-like
## target: array-like
## k: int
## http://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.SelectKBest.html

基于树模型

import numpy as np
from sklearn import feature_selection
from sklearn.ensemble import GradientBoostingClassifier

matrix = np.array(X)
target = np.array(target)
temp = feature_selection.SelectFromModel(GradientBoostingClassifier()).fit(matrix, target)
indx = temp._get_support_mask().tolist()
scores = get_importance(temp.estimator_).tolist()
result = temp.transform(matrix).tolist()
return scores, indx, result

## X: array-like
## target: array-like
## http://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.SelectFromModel.html

卡方检验法

import numpy as np
from sklearn import feature_selection
from sklearn.feature_selection import chi2

matrix = np.array(X)
target = np.array(target)
temp = feature_selection.SelectKBest(chi2, k=k).fit(matrix, target)
scores = temp.scores_.tolist()
indx = temp.get_support().tolist()
result = temp.transform(matrix).tolist()
return scores, indx, result

## X: array-like
## target: array-like
## k: int
## http://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.SelectKBest.html

基于惩罚值

import numpy as np
from sklearn import feature_selection
from sklearn.linear_model import LogisticRegression

matrix = np.array(arr0)
target = np.array(target)
temp = feature_selection.SelectFromModel(LogisticRegression(penalty="l1", C=0.1)).fit(matrix, target)
indx = temp._get_support_mask().tolist()
scores = get_importance(temp.estimator_).tolist()
result = temp.transform(matrix).tolist()
return scores, indx, result

## X: array-like
## target: array-like
## http://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.SelectFromModel.html

降维

PCA主成分分析算法

import numpy as np
from sklearn.decomposition import PCA

matrix = np.array(X)
pca = PCA(n_components='mle', svd_solver='auto').fit(matrix)
result = pca.transform(matrix)
label = result.tolist()
return label, pca.components_.tolist(), pca.explained_variance_.tolist(), pca.explained_variance_ratio_.tolist(), pca.mean_.tolist(), pca.noise_variance_

哑变量

import numpy as np
from sklearn import preprocessing

raw_result = preprocessing.OneHotEncoder().fit_transform(np.array(X))
result = raw_result.tolist()

## X: array-like
## http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.OneHotEncoder.html

TSNE-t 分布邻域嵌入算法

import numpy as np
from sklearn.manifold import TSNE

matrix = np.array(X)
t_sne = TSNE(n_components=2, random_state=0)
np.set_printoptions(suppress=True)
result = t_sne.fit(matrix)
kl_divergence = result.kl_divergence_
label = t_sne.fit_transform(matrix).tolist()

return label, kl_divergence
## X: array-like
## http://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html

线性判别分析法(LDA)

from sklearn.lda import LDA
import numpy as np
from sklearn import preprocessing

matrix = np.array(X)
target = np.array(target)
temp = LDA(n_components=n_components).fit(matrix, target)
coef = temp.coef_
mean = temp.means_
priors = temp.priors_
scalings = temp.scalings_
xbar = temp.xbar_
label = temp.transform(matrix).tolist()
return label, coef.tolist(), mean.tolist(), priors.tolist(), scalings.tolist(), xbar.tolist()

## X: array-like
## target: array-like
## n_components: int
## http://scikit-learn.org/0.15/modules/generated/sklearn.lda.LDA.html

分类器

随机梯度下降(SGD)

## http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.SGDClassifier.html#sklearn.linear_model.SGDClassifier
from sklearn.linear_model import SGDClassifier

clf = SGDClassifier(loss="hinge", penalty="l2")
clf.fit(X, y)

## 数组 X shape:[n_samples, n_features]
## 数组 y shape:[n_samples]

逻辑回归(LR)

## http://scikit-learn.org/stable/modules/linear_model.html
from sklearn import linear_model

lr =linear_model.LogisticRegression()  ## penalty : str, ‘l1’ or ‘l2’, default: ‘l2’
lr.fit(X, y)

分类树(Tree)

#http://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeClassifier.html
from sklearn.tree import DecisionTreeClassifier

## criterion = "gini" (CART) or "entropy" (ID3)
clf = DecisionTreeClassifier(criterion = 'entropy' ,random_state = 0)
clf.fit(X,y)

随机森林(Random Forest)

## http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html
from sklearn.ensemble import RandomForestClassifier
import numpy as np

## criterion: str, 'gini' for Gini impurity (Default) and “entropy” for the information gain.
clf = RandomForestClassifier(criterion='gini')  
## X: array-like, shape = [n_samples, n_features]
## y: array-like, shape = [n_samples] or [n_samples, n_outputs]
clf.fit(np.array(X), np.array(y))

支持向量机(SVM)

## http://scikit-learn.org/dev/modules/generated/sklearn.svm.SVC.html#sklearn.svm.SVC
from sklearn import svm
import numpy as np

## Classifier Option 1: SVC()
clf = svm.SVC()       ## kernel = 'linear' or 'rbf' (default) or 'poly' or custom kernels; penalty C = 1.0 (default)
## Option 2: NuSVC()
## clf = svm.NuSVC() 
## Option 3: LinearSVC()
## clf = svm.LinearSVC()     ## penalty : str, ‘l1’ or ‘l2’ (default=’l2’)
clf.fit(X, y)                ## X shape = [n_samples, n_features], y shape = [n_samples] or [n_samples, n_outputs]

## print(clf.support_vectors_) ## get support vectors
## print(clf.support_)         ## get indeices of support vectors
## print(clf.n_support_)       ## get number of support vectors for each class

mean_accuracy = clf.score(X,y)
print("Accuracy: %.3f"%(mean_accuracy))

朴素贝叶斯(Naive Bayesian)

## http://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.GaussianNB.html
from sklearn.naive_bayes import GaussianNB

gnb = GaussianNB()
## predict(test_set或者train_set) 值为你需要预测的数据集
y_pred = gnb.fit(X, y).predict(X)
print("Number of mislabeled points out of a total %d points : %d" % (X.shape[0],(y != y_pred).sum()))

AdaBoostClassifier

## 决策树集成
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier

adaboost_dt_clf = AdaBoostClassifier(
                                    DecisionTreeClassifier(
                                        max_depth=2,   ## 决策树最大深度，默认可不输入即不限制子树深度
                                        min_samples_split=20, ## 内部结点再划分所需最小样本数，默认值为2，若样本量不大，无需更改，反之增大
                                        min_samples_leaf=5    ## 叶子节点最少样本数,默认值为1，若样本量不大，无需更改，反之增大
                                        ),
                                    algorithm="SAMME", ## boosting 算法 {‘SAMME’, ‘SAMME.R’}, 默认为后者
                                    n_estimators=200,  ## 最多200个弱分类器，默认值为50
                                    learning_rate=0.8  ## 学习率，默认值为1
                                     )
adaboost_dt_clf.fit(X,y)

GBDT(梯度加速决策树)

## http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html#sklearn.ensemble.GradientBoostingClassifier
from sklearn.ensemble import GradientBoostingClassifier

gbdt = GradientBoostingClassifier(max_depth=4,   ## 决策树最大深度，默认可不输入，即不限制子树深度
                                max_features="auto",  ## 寻找最优分割的特征数量，可为int,float,"auto","sqrt","log2",None:
                                n_estimators=100 ## Boosting阶段的数量，默认值为100。
                                )
gbdt.fit(X,y)

K-Means

## 聚类
## http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html
from sklearn.cluster import KMeans

kmeans = KMeans(
                n_clusters = 2, ## 簇的个数，默认值为8
                random_state=0  
                ).fit(X)

print(kmeans.labels_)
print("K Clusters Centroids:\n", kmeans.cluster_centers_)

使用 keras 做分类

from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import Adam
from keras.utils import np_utils

training_epochs = 200 #训练次数，总体数据需要循环多少次
batch_size = 10  

model = Sequential()
input = X.shape[1]
## 隐藏层128
model.add(Dense(128, input_shape=(input,)))
model.add(Activation('relu'))
## Dropout层用于防止过拟合
model.add(Dropout(0.2))
## 隐藏层128
model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(0.2))
## 没有激活函数用于输出层，二分类问题，用sigmoid激活函数进行变换，多分类用softmax。
model.add(Dense(1))
model.add(Activation('sigmoid'))
## 使用高效的 ADAM 优化算法以，二分类损失函数binary_crossentropy，多分类的损失函数categorical_crossentropy
model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])

history = model.fit(X_train, y_train, validation_data=(X_test,y_test), epochs=training_epochs, batch_size=32)

线性回归

使用sklearn

import sklearn
sklearn.linear_model.LinearRegression()

## http://scikit-learn.org/stable/modules/linear_model.html

高度专门化的线性回归函数(scipy.stats)

import scipy
from scipy import stats

stats.linregress(x, y=none)

## x, y : array_like
## https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.linregress.html

使用Statsmodels的模型

import statsmodels.api as sm
results = sm.OLS(y, X).fit()

## y: matrix
## X: constant
## http://www.statsmodels.org/dev/index.html

一般的最小二乘多项式拟合(numpy)

import numpy as np
np.polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False)
## x : array_like, shape (M,)
## y : array_like, shape (M,) or (M, K)
## deg : int
## https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.polyfit.html

用矩阵因式分解计算最小二乘(numpy.linalg.lstsq)

import numpy
from numpy import linalg

linalg.lstsq(a, b, rcond=-1)

## a : (M, N) array_like
## b : {(M,), (M, K)} array_like
## https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.lstsq.html

更通用的最小二乘极小化(scipy.optimize)

import scipy
from scipy import optimize

optimize.curve_fit(f, xdata, ydata, p0=None, sigma=None, absolute_sigma=False, check_finite=True, bounds=(-inf, inf), method=None, jac=None, **kwargs)

## f : callable
## xdata : An M-length sequence or an (k,M)-shaped array for functions with k predictors
## ydata : M-length sequence
## https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.curve_fit.html

使用 keras 做线性回归

from keras.models import Sequential #顺序模型
from keras.layers import Dense

n_hidden_1 = 64 #隐藏层1的神经元个数
n_hidden_2 = 64 #隐藏层2的神经元个数
n_input = 13 #输入层的个数，也就是特征数
n_classes = 1 #输出层的个数
training_epochs = 200 #训练次数，总体数据需要循环多少次
batch_size = 10  

model = Sequential()#先建立一个顺序模型
#向顺序模型里加入第一个隐藏层，第一层一定要有一个输入数据的大小，需要有input_shape参数
#model.add(Dense(n_hidden_1, activation='relu', input_shape=(n_input,)))
model.add(Dense(n_hidden_1, activation='relu', input_dim=n_input))
model.add(Dense(n_hidden_2, activation='relu'))
model.add(Dense(n_classes))

model.compile(loss='mse', optimizer='rmsprop', metrics=['mae',r2])

history = model.fit(X_train, y_train, validation_data=(X_test,y_test), epochs=training_epochs, batch_size=batch_size)

愿无知者有力，愿有力者前行。
我是@饼干哥哥，持续为您打造数字化时代的分析能力。

我开发了数据分析学习网站：www.qschatdata.tech
同时运营公众号：饼干哥哥数据分析，关注回复资料，领取学习礼包。

标签：数据分析,df,Python,代码,##,np,import,my,col
From： https://blog.51cto.com/u_15927359/6961665

数据分析/数据科学Python常用代码块

换源_python

模型服务

keras 部署服务

可部署为服务的 Python 脚本

SQL操作

连接 PostgreSQL

单表导出数据

使用 WHERE 语句过滤

使用 HAVING 语句过滤

取出前N条数据

多表导出数据

使用聚合函数

使用 Subquery

使用 WITH 语句

通用条件表达式 (类似if-else)

查看数据库中所有表名

窗口函数

查看表内字段类型

基本操作

装包换源-Python

静默安装-Python

皮尔森相关系数

移动平均数

列的分位数

多重聚合(组数据)

以列值排序

用户定义方程(组数据)

标准差

在聚合的dataframe上使用用户定义方程

所有列(列的数据类型为数值型)

以频率降序排序

一些列的单一属性(如最大值， 列的类型为数值型)

最大互信息数

平均数

组数据的基本信息

行列数

相关性

中位数

数据组的遍历

协方差

方差

K平均数算法

所有列的单一属性(如最大值， 列的类型为数值型)

获得类别字段的频数

查看缺失情况

查看字段包含unique值的程度

新建列

使用复杂的方程设值

标准聚合(使用groupby)

使用自定义方程设值

用户定义的聚合方程(使用groupby)

缺失值处理

用一个值填补多列的缺失值

用一个值填补一列的缺失值

去除在指定列中带有缺失值的数据条目

用最后一个或缺失值的下一个数据填补缺失值

用一个由聚合得出的值填补缺失值

以列为单位审查数据集

去除所有任何缺失值的数据条目

Pandas基本操作

计算频率来转置表

水平连接数据

将dataframe的index重置成标准值

以元组(tuple)形式按行写入数据

数据重构

用数据的位置获取数据

从列新建虚数据

合并有相同名称的列的两个dataframe

合并两个dataframe没有相同名称的列

分组数据

使用目标值的位置定位替换值

分簇

获取元组(tuple)的迭代器

获取文档

复制数据

重命名所有列

设置聚合方程进行交叉制表

高级索引

使用某种法则替换列中的值

一些列的单一属性(如最大值，列的类型为数值型)

所有列的单一属性(如最大值，列的类型为数值型)