机器学习图像的情绪分析

标签：plt 机器 48 img 情绪 train 图像 import model

一、选题的背景

　　人脸面部表情活动时刻存在于人们的生活交流过程中，它是人类情绪表达的一种途径，是人类情感信息传递的一种方式，还是人类进行非语言交流的一种渠道，所以它具有举足轻重的作用。据心理学家A. Mehrabian研究表明，日常交流信息的传递主要包括三种方式：语言传递、声音传递和面部表情传递。在这三种方式中，通过面部表情所传递的信息量高达信息总量的55%。也就是说，人脸面部表情携带了丰富的情感信息，对面部表情的判别分析是获得这些信息的重要途径。通过观察分析人脸面部表情，可以推测人们的心理活动，预测人体行为，进而做出人类所期待的响应。因此，许多学者提出了人脸面部表情识别技术。

 二、机器学习案例设计方案

　　1.数据集包含 35685 个 48x48 像素灰度人脸图像示例，分为训练数据集和测试数据集。图像根据面部表情中显示的情绪（快乐、中立、悲伤、愤怒、惊讶、厌恶、恐惧）进行分类。

　　2.卷积神经网络（Convolutional Neural Networks, CNN）是一类包含卷积计算且具有深度结构的前馈神经网络（Feedforward Neural Networks），是深度学习（deep learning）的代表算法之一 。卷积神经网络具有表征学习（representation learning）能力，能够按其阶层结构对输入信息进行平移不变分类（shift-invariant classification），因此也被称为“平移不变人工神经网络（Shift-Invariant Artificial Neural Networks, SIANN）” 。

　　3.在处理图像中许多的情绪表情有许多的相似处，容易造成检测出现错误。通过增加学习的方式，提高检测的准确性。

三、机器学习的实现步骤

#需要的库
import os
os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"
import PIL
import cv2
import glob
import pathlib
import zipfile
import numpy as np
from fastai import *
from PIL import Image
import tensorflow as tf
from fastai.vision import *
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.layers import *
from matplotlib import pyplot as plt
from keras.preprocessing import image
from keras.preprocessing.image import *
from tensorflow.keras.models import Sequential
from fastai.metrics import error_rate, accuracy
from keras.preprocessing.image import ImageDataGenerator
from torch.utils.data import Dataset, DataLoader
from keras.callbacks import ModelCheckpoint
from sklearn.utils import class_weight
from tensorflow.keras.models import load_model
from sklearn.metrics import classification_report, confusion_matrix

设置路径

代码展示：

data_dir = 'E:/face/'
train_path = data_dir + 'train/'
test_path = data_dir + 'test/'

定义函数，展示3x3的不同情绪图像

代码展示：

def display_images(emotion):
    plt.figure(figsize=(10,10))
    for i in range(1, 10, 1):
        plt.subplot(3,3,i) #为每个标签形成3x3网格
        #img = image.load_img(train_path + '/' + emotion + "/" + os.listdir(train_path + '/' + emotion)[i], target_size=(48, 48)) #load images
        img = image.load_img(train_path + '/' + emotion + "/" + os.listdir(train_path + '/' + emotion)[i], target_size=(48, 48)) #load images
        plt.title(emotion) #显示图像标签
        plt.imshow(img) #显示图像
        plt.tight_layout() #格式化图像
    plt.show()
for i in os.listdir(train_path):
    display_images(i)

效果展示：

 

定义函数，显示标签

代码展示：

def mylistdir(directory):
    filelist = os.listdir(directory)
    return [x for x in filelist
            if not (x.startswith('.'))]
#显示类标签
labels = os.listdir(train_path)
print("七种情绪:")
labels

效果展示：

 

 

统计数据集图片

 

代码展示：

file_count = len(list(pathlib.Path(data_dir).glob('*/*/*.png')))
print('Total image count:', file_count)

file_count = len(list(pathlib.Path(train_path).glob('*/*.png')))
print('Total training images:', file_count)

file_count = len(list(pathlib.Path(test_path).glob('*/*.png')))
print('Total testing images:', file_count)

效果展示：

 

 

显示7个情绪标签和每个情绪的示例图像

代码展示：

#显示7个情绪标签和每个情绪的示例图像
fig, axes = plt.subplots(1, 7, figsize=(20,20))

for i in range(7):

    ex_image = train_path + labels[i]+ '/' + mylistdir(train_path + labels[i]+'/')[0]
    axes[i].imshow(plt.imread(ex_image))
    axes[i].set_title(labels[i])

plt.show()

效果展示：

 

 

 代码展示：

#打印图像标签分发
for i in labels:
    print(i, '\nTrain: ' + str(len(os.listdir(train_path + i +'/')))+ ' images' +'\nTest: ' + str(len(os.listdir(test_path+i+'/')))+' images\n')

效果展示：

 

 

 代码展示：

train_dist = np.array([len(os.listdir(train_path+i+'/')) for i in labels])
test_dist = np.array([len(os.listdir(test_path+i+'/')) for i in labels])
x = labels

plt.figure(figsize=(20,10))
plt.suptitle('Emotion Distribution', fontsize=25)

ax1 = plt.subplot(1,2,1)
ax1.set_title('Training Set')
plt.xlabel('Emotion Class', fontsize=14)
plt.ylabel('Number of Images', fontsize=14)
plt.bar(x, train_dist, color='blue')

ax2 = plt.subplot(1, 2, 2)
ax2.set_title('Test Set')
plt.xlabel('Emotion Class', fontsize=14)
plt.ylabel('Number of Images', fontsize=14)
plt.bar(x, test_dist, color='green')
plt.show()

效果展示：

 

 

 代码展示：

#定义数据扩充
train_datagen = ImageDataGenerator(
    rescale=1.0/255,
    zoom_range= 0.2,
    horizontal_flip=True,
    shear_range=0.2,
    validation_split=0.2)

test_datagen = ImageDataGenerator(rescale=1.0/255)

#将图像读取到数据扩充
#生成批量扩增数据
train = train_datagen.flow_from_directory(
    train_path,
    subset='training',
    color_mode = 'grayscale',
    target_size = (48, 48),
    batch_size = 64,
    shuffle=True,
    class_mode = 'categorical')

valid = train_datagen.flow_from_directory(
    train_path,
    subset='validation',
    color_mode = 'grayscale',
    target_size = (48, 48),
    batch_size = 64,
    shuffle=True,
    class_mode = 'categorical')

test = test_datagen.flow_from_directory(
    test_path,
    color_mode = 'grayscale',
    target_size = (48, 48),
    batch_size = 64,
    shuffle=False,
    class_mode = 'categorical')

效果展示：

 

代码展示：

train.class_indices

效果展示：

 

 

 代码展示：

#初始化我们的模型
model = tf.keras.Sequential()

#输入层
model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(48, 48, 1), padding='same'))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2))) #减少维度的最大值池化
model.add(Dropout(0.25)) #test

model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2))) 
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(Dropout(0.25))

model.add(Flatten()) 

model.add(Dense(1024, activation = 'relu'))
model.add(Dropout(0.5))
model.add(Dense(7, activation = 'softmax'))
model.summary()

效果展示：

 

代码展示：

from sklearn.utils import class_weight
class_weights = class_weight.compute_class_weight(
    class_weight='balanced', 
    classes=np.unique(train.classes), 
    y=train.classes)
class_weights = dict(zip(np.unique(train.classes),class_weights))

filepath = 'my_best_model.epoch{epoch:02d}-loss{val_loss:.2f}.hdf5' #使用自定义名称将最佳模型保存到此位置
callbacks = [tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2,patience=5, min_delta=0.0001, verbose=1),
             tf.keras.callbacks.ModelCheckpoint(filepath=filepath, monitor='val_loss',verbose=1, save_best_only=True, mode='min'),
             tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1,restore_best_weights=True)]

model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001),
              loss='categorical_crossentropy',
              metrics=['accuracy'])
emotion = model.fit(train, validation_data=valid, epochs=50, callbacks=callbacks, class_weight=class_weights)

效果展示：

 

 

 代码展示：

loss,acc = model.evaluate(test,verbose=2)

效果展示：

 

 

 代码展示：

#保存模型/权重
model.save('emotion-detection.h5')
model.save_weights('emotion_weight.h5')
np.save("history", emotion.history)
print('保存成功！.')

代码展示：

plt.figure(figsize=(20,10))
plt.suptitle('Loss & Accuracy Over Time', fontsize=25)

plt.subplot(1, 2, 1)
plt.xlabel('Epoch', fontsize=18)
plt.ylabel('Loss', fontsize=18)
plt.plot(emotion.history['loss'], label='Training Loss')
plt.plot(emotion.history['val_loss'], label='Validation Loss')
plt.legend(loc='upper right')

plt.subplot(1, 2, 2)
plt.xlabel('Epoch', fontsize=18)
plt.ylabel('Accuracy', fontsize=16)
plt.plot(emotion.history['accuracy'], label='Training Accuracy')
plt.plot(emotion.history['val_accuracy'], label='Validation Accuracy')
plt.legend(loc='upper left')
plt.show()

效果展示：

 

 

 代码展示：

train_loss, train_acc = model.evaluate(train)
test_loss, test_acc = model.evaluate(test)

效果展示：

 

 

 代码展示：

mod = load_model('my_best_model.epoch28-loss1.11.hdf5')
y_pred = model.predict(train)
y_pred = np.argmax(y_pred, axis=1)
class_labels = test.class_indices
class_labels = {v:k for k,v in class_labels.items()}

print('Confusion Matrix')
print(confusion_matrix)
print('Classification Report')
target_names = list(class_labels.values())
print(classification_report(train.classes, y_pred, target_names=target_names))

效果展示：

 

 

 代码展示：

img = image.load_img(test_path+"/surprised/im30.png",target_size = (48,48),color_mode = "grayscale")
img = np.array(img)
plt.imshow(img)

效果展示：

 

 

 代码展示：

labels = sorted(labels)
img = np.expand_dims(img,axis = 0) #reshapes to 1,48,48
img = img.reshape(1,48,48,1)
img=tf.cast(img,tf.float64)
result = model.predict(img) #用模型预测图像的情感
result = list(result[0])
print(result)
print(train.class_indices)

效果展示：

 

 代码展示：

img_index = result.index(max(result))
print('Prediction:',labels[img_index])

效果展示：

 

惊讶测试：

#惊讶的脸
img = image.load_img(test_path+"/fearful/im30.png", target_size=(48,48),color_mode='grayscale')
img = np.array(img)
plt.imshow(img)

效果展示：

 

 代码展示：

img = np.expand_dims(img,axis = 0) #reshapes to 1,48,48
img = img.reshape(1,48,48,1)
img=tf.cast(img,tf.float32)
result = model.predict(img) #用模型预测图像的情感
result = list(result[0])

img_index = result.index(max(result))
print('Prediction:',labels[img_index])

plt.figure(figsize=(10,5))
plt.suptitle('Emotion Prediction')

ax1 = plt.subplot()
plt.xlabel('Emotion', fontsize=14)
plt.ylabel('Certainty', fontsize=14)


plt.bar(labels, np.array(result), color='blue')

效果展示：

 代码展示：

img = image.load_img('E:/demo.png', target_size=(48,48),color_mode='grayscale')
img = np.array(img)
plt.imshow(img)

效果展示：

 

代码展示：

img = np.expand_dims(img,axis = 0) #reshapes to 1,48,48
img = img.reshape(1,48,48,1)
img=tf.cast(img,tf.float64)
result = model.predict(img) #用模型预测图像的情感
result = list(result[0])

img_index = result.index(max(result))
print('Prediction:',labels[img_index])

plt.figure(figsize=(10,5))
plt.suptitle('Emotion Prediction')

ax1 = plt.subplot()
plt.xlabel('Emotion', fontsize=14)
plt.ylabel('Certainty', fontsize=14)


plt.bar(labels, np.array(result), color='blue')

效果展示：

 

 

 四、总结

　　机器学习的应用领域非常广泛，学习难度也比较大，说白了就是对于给定的数据进行处理，利用计算机的能力来输出一个output，判断是否符合自己想要的结果，然后不断的优化、完善，最终形成自己的项目。

　　在做这个项目的时候还是遇到了许多问题，起先就是需要的库版本不对应，在通过百度、博客等方法后成功解决了。然后在项目实施的过程中，还存在着输出结果还不能符合自己的需要，后期通过增加训练方法，优化图像处理方面的能力。

 

数据集：

https://www.heywhale.com/mw/dataset/639b611e6d0ae1910070d39d/file

完整代码：

#需要的库
import os
os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"
import PIL
import cv2
import glob
import pathlib
import zipfile
import numpy as np
from fastai import *
from PIL import Image
import tensorflow as tf
from fastai.vision import *
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.layers import *
from matplotlib import pyplot as plt
from keras.preprocessing import image
from keras.preprocessing.image import *
from tensorflow.keras.models import Sequential
from fastai.metrics import error_rate, accuracy
from keras.preprocessing.image import ImageDataGenerator
from torch.utils.data import Dataset, DataLoader
from keras.callbacks import ModelCheckpoint
from sklearn.utils import class_weight
from tensorflow.keras.models import load_model
from sklearn.metrics import classification_report, confusion_matrix
data_dir = 'E:/face/'
train_path = data_dir + 'train/'
test_path = data_dir + 'test/'
def display_images(emotion):
    plt.figure(figsize=(10,10))
    for i in range(1, 10, 1):
        plt.subplot(3,3,i) #为每个标签形成3x3网格
        #img = image.load_img(train_path + '/' + emotion + "/" + os.listdir(train_path + '/' + emotion)[i], target_size=(48, 48)) #load images
        img = image.load_img(train_path + '/' + emotion + "/" + os.listdir(train_path + '/' + emotion)[i], target_size=(48, 48)) #load images
        plt.title(emotion) #显示图像标签
        plt.imshow(img) #显示图像
        plt.tight_layout() #格式化图像
    plt.show()
for i in os.listdir(train_path):
    display_images(i)
def mylistdir(directory):
    filelist = os.listdir(directory)
    return [x for x in filelist
            if not (x.startswith('.'))]
#显示类标签
labels = os.listdir(train_path)
print("七种情绪:")
labels
file_count = len(list(pathlib.Path(data_dir).glob('*/*/*.png')))
print('Total image count:', file_count)

file_count = len(list(pathlib.Path(train_path).glob('*/*.png')))
print('Total training images:', file_count)

file_count = len(list(pathlib.Path(test_path).glob('*/*.png')))
print('Total testing images:', file_count)
#显示7个情绪标签和每个情绪的示例图像
fig, axes = plt.subplots(1, 7, figsize=(20,20))

for i in range(7):

    ex_image = train_path + labels[i]+ '/' + mylistdir(train_path + labels[i]+'/')[0]
    axes[i].imshow(plt.imread(ex_image))
    axes[i].set_title(labels[i])

plt.show()
#打印图像标签分发
for i in labels:
    print(i, '\nTrain: ' + str(len(os.listdir(train_path + i +'/')))+ ' images' +'\nTest: ' + str(len(os.listdir(test_path+i+'/')))+' images\n')
train_dist = np.array([len(os.listdir(train_path+i+'/')) for i in labels])
test_dist = np.array([len(os.listdir(test_path+i+'/')) for i in labels])
x = labels

plt.figure(figsize=(20,10))
plt.suptitle('Emotion Distribution', fontsize=25)

ax1 = plt.subplot(1,2,1)
ax1.set_title('Training Set')
plt.xlabel('Emotion Class', fontsize=14)
plt.ylabel('Number of Images', fontsize=14)
plt.bar(x, train_dist, color='blue')

ax2 = plt.subplot(1, 2, 2)
ax2.set_title('Test Set')
plt.xlabel('Emotion Class', fontsize=14)
plt.ylabel('Number of Images', fontsize=14)
plt.bar(x, test_dist, color='green')
plt.show()
#定义数据扩充
train_datagen = ImageDataGenerator(
    rescale=1.0/255,
    zoom_range= 0.2,
    horizontal_flip=True,
    shear_range=0.2,
    validation_split=0.2)

test_datagen = ImageDataGenerator(rescale=1.0/255)

#将图像读取到数据扩充
#生成批量扩增数据
train = train_datagen.flow_from_directory(
    train_path,
    subset='training',
    color_mode = 'grayscale',
    target_size = (48, 48),
    batch_size = 64,
    shuffle=True,
    class_mode = 'categorical')

valid = train_datagen.flow_from_directory(
    train_path,
    subset='validation',
    color_mode = 'grayscale',
    target_size = (48, 48),
    batch_size = 64,
    shuffle=True,
    class_mode = 'categorical')

test = test_datagen.flow_from_directory(
    test_path,
    color_mode = 'grayscale',
    target_size = (48, 48),
    batch_size = 64,
    shuffle=False,
    class_mode = 'categorical')

train.class_indices
#初始化我们的模型
model = tf.keras.Sequential()

#输入层
model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(48, 48, 1), padding='same'))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2))) #减少维度的最大值池化
model.add(Dropout(0.25)) #test

model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2))) 
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(Dropout(0.25))

model.add(Flatten()) 

model.add(Dense(1024, activation = 'relu'))
model.add(Dropout(0.5))
model.add(Dense(7, activation = 'softmax'))
model.summary()
class_weights = class_weight.compute_class_weight(
    class_weight='balanced', 
    classes=np.unique(train.classes), 
    y=train.classes)
class_weights = dict(zip(np.unique(train.classes),class_weights))

filepath = 'my_best_model.epoch{epoch:02d}-loss{val_loss:.2f}.hdf5' #使用自定义名称将最佳模型保存到此位置
callbacks = [tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2,patience=5, min_delta=0.0001, verbose=1),
             tf.keras.callbacks.ModelCheckpoint(filepath=filepath, monitor='val_loss',verbose=1, save_best_only=True, mode='min'),
             tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1,restore_best_weights=True)]

model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001),
              loss='categorical_crossentropy',
              metrics=['accuracy'])
emotion = model.fit(train, validation_data=valid, epochs=50, callbacks=callbacks, class_weight=class_weights)
loss,acc = model.evaluate(test,verbose=2)
#保存模型/权重
model.save('emotion-detection.h5')
model.save_weights('emotion_weight.h5')
np.save("history", emotion.history)
print('保存成功！.')
plt.figure(figsize=(20,10))
plt.suptitle('Loss & Accuracy Over Time', fontsize=25)

plt.subplot(1, 2, 1)
plt.xlabel('Epoch', fontsize=18)
plt.ylabel('Loss', fontsize=18)
plt.plot(emotion.history['loss'], label='Training Loss')
plt.plot(emotion.history['val_loss'], label='Validation Loss')
plt.legend(loc='upper right')

plt.subplot(1, 2, 2)
plt.xlabel('Epoch', fontsize=18)
plt.ylabel('Accuracy', fontsize=16)
plt.plot(emotion.history['accuracy'], label='Training Accuracy')
plt.plot(emotion.history['val_accuracy'], label='Validation Accuracy')
plt.legend(loc='upper left')
plt.show()
train_loss, train_acc = model.evaluate(train)
test_loss, test_acc = model.evaluate(test)
mod = load_model('my_best_model.epoch28-loss1.11.hdf5')
y_pred = model.predict(train)
y_pred = np.argmax(y_pred, axis=1)
class_labels = test.class_indices
class_labels = {v:k for k,v in class_labels.items()}

print('Confusion Matrix')
print(confusion_matrix)
print('Classification Report')
target_names = list(class_labels.values())
print(classification_report(train.classes, y_pred, target_names=target_names))
img = image.load_img(test_path+"/surprised/im30.png",target_size = (48,48),color_mode = "grayscale")
img = np.array(img)
plt.imshow(img)
labels = sorted(labels)
img = np.expand_dims(img,axis = 0) #reshapes to 1,48,48
img = img.reshape(1,48,48,1)
img=tf.cast(img,tf.float64)
result = model.predict(img) #用模型预测图像的情感
result = list(result[0])
print(result)
print(train.class_indices)
img_index = result.index(max(result))
print('Prediction:',labels[img_index])
#惊讶的脸
img = image.load_img(test_path+"/fearful/im30.png", target_size=(48,48),color_mode='grayscale')
img = np.array(img)
plt.imshow(img)
img = np.expand_dims(img,axis = 0) #reshapes to 1,48,48
img = img.reshape(1,48,48,1)
img=tf.cast(img,tf.float32)
result = model.predict(img) #用模型预测图像的情感
result = list(result[0])

img_index = result.index(max(result))
print('Prediction:',labels[img_index])

plt.figure(figsize=(10,5))
plt.suptitle('Emotion Prediction')

ax1 = plt.subplot()
plt.xlabel('Emotion', fontsize=14)
plt.ylabel('Certainty', fontsize=14)


plt.bar(labels, np.array(result), color='blue')
img = image.load_img('E:/demo.png', target_size=(48,48),color_mode='grayscale')
img = np.array(img)
plt.imshow(img)
img = np.expand_dims(img,axis = 0) #reshapes to 1,48,48
img = img.reshape(1,48,48,1)
img=tf.cast(img,tf.float64)
result = model.predict(img) #用模型预测图像的情感
result = list(result[0])

img_index = result.index(max(result))
print('Prediction:',labels[img_index])

plt.figure(figsize=(10,5))
plt.suptitle('Emotion Prediction')

ax1 = plt.subplot()
plt.xlabel('Emotion', fontsize=14)
plt.ylabel('Certainty', fontsize=14)


plt.bar(labels, np.array(result), color='blue')

 

标签：plt,机器,48,img,情绪,train,图像,import,model
From： https://www.cnblogs.com/zhangbiqia2103840118/p/16987250.html