一、选题的背景
人脸面部表情活动时刻存在于人们的生活交流过程中,它是人类情绪表达的一种途径,是人类情感信息传递的一种方式,还是人类进行非语言交流的一种渠道,所以它具有举足轻重的作用。据心理学家A. Mehrabian研究表明,日常交流信息的传递主要包括三种方式:语言传递、声音传递和面部表情传递。在这三种方式中,通过面部表情所传递的信息量高达信息总量的55%。也就是说,人脸面部表情携带了丰富的情感信息,对面部表情的判别分析是获得这些信息的重要途径。通过观察分析人脸面部表情,可以推测人们的心理活动,预测人体行为,进而做出人类所期待的响应。因此,许多学者提出了人脸面部表情识别技术。
二、机器学习案例设计方案
1.数据集包含 35685 个 48x48 像素灰度人脸图像示例,分为训练数据集和测试数据集。图像根据面部表情中显示的情绪(快乐、中立、悲伤、愤怒、惊讶、厌恶、恐惧)进行分类。
2.卷积神经网络(Convolutional Neural Networks, CNN)是一类包含卷积计算且具有深度结构的前馈神经网络(Feedforward Neural Networks),是深度学习(deep learning)的代表算法之一 。卷积神经网络具有表征学习(representation learning)能力,能够按其阶层结构对输入信息进行平移不变分类(shift-invariant classification),因此也被称为“平移不变人工神经网络(Shift-Invariant Artificial Neural Networks, SIANN)” 。
3.在处理图像中许多的情绪表情有许多的相似处,容易造成检测出现错误。通过增加学习的方式,提高检测的准确性。
三、机器学习的实现步骤
#需要的库 import os os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE" import PIL import cv2 import glob import pathlib import zipfile import numpy as np from fastai import * from PIL import Image import tensorflow as tf from fastai.vision import * from tensorflow import keras from tensorflow.keras import layers from tensorflow.keras.layers import * from matplotlib import pyplot as plt from keras.preprocessing import image from keras.preprocessing.image import * from tensorflow.keras.models import Sequential from fastai.metrics import error_rate, accuracy from keras.preprocessing.image import ImageDataGenerator from torch.utils.data import Dataset, DataLoader from keras.callbacks import ModelCheckpoint from sklearn.utils import class_weight from tensorflow.keras.models import load_model from sklearn.metrics import classification_report, confusion_matrix
设置路径
代码展示:
data_dir = 'E:/face/' train_path = data_dir + 'train/' test_path = data_dir + 'test/'
定义函数,展示3x3的不同情绪图像
代码展示:
def display_images(emotion): plt.figure(figsize=(10,10)) for i in range(1, 10, 1): plt.subplot(3,3,i) #为每个标签形成3x3网格 #img = image.load_img(train_path + '/' + emotion + "/" + os.listdir(train_path + '/' + emotion)[i], target_size=(48, 48)) #load images img = image.load_img(train_path + '/' + emotion + "/" + os.listdir(train_path + '/' + emotion)[i], target_size=(48, 48)) #load images plt.title(emotion) #显示图像标签 plt.imshow(img) #显示图像 plt.tight_layout() #格式化图像 plt.show() for i in os.listdir(train_path): display_images(i)
效果展示:
定义函数,显示标签
代码展示:
def mylistdir(directory): filelist = os.listdir(directory) return [x for x in filelist if not (x.startswith('.'))] #显示类标签 labels = os.listdir(train_path) print("七种情绪:") labels
效果展示:
统计数据集图片
代码展示:
file_count = len(list(pathlib.Path(data_dir).glob('*/*/*.png'))) print('Total image count:', file_count) file_count = len(list(pathlib.Path(train_path).glob('*/*.png'))) print('Total training images:', file_count) file_count = len(list(pathlib.Path(test_path).glob('*/*.png'))) print('Total testing images:', file_count)
效果展示:
显示7个情绪标签和每个情绪的示例图像
代码展示:
#显示7个情绪标签和每个情绪的示例图像 fig, axes = plt.subplots(1, 7, figsize=(20,20)) for i in range(7): ex_image = train_path + labels[i]+ '/' + mylistdir(train_path + labels[i]+'/')[0] axes[i].imshow(plt.imread(ex_image)) axes[i].set_title(labels[i]) plt.show()
效果展示:
代码展示:
#打印图像标签分发 for i in labels: print(i, '\nTrain: ' + str(len(os.listdir(train_path + i +'/')))+ ' images' +'\nTest: ' + str(len(os.listdir(test_path+i+'/')))+' images\n')
效果展示:
代码展示:
train_dist = np.array([len(os.listdir(train_path+i+'/')) for i in labels]) test_dist = np.array([len(os.listdir(test_path+i+'/')) for i in labels]) x = labels plt.figure(figsize=(20,10)) plt.suptitle('Emotion Distribution', fontsize=25) ax1 = plt.subplot(1,2,1) ax1.set_title('Training Set') plt.xlabel('Emotion Class', fontsize=14) plt.ylabel('Number of Images', fontsize=14) plt.bar(x, train_dist, color='blue') ax2 = plt.subplot(1, 2, 2) ax2.set_title('Test Set') plt.xlabel('Emotion Class', fontsize=14) plt.ylabel('Number of Images', fontsize=14) plt.bar(x, test_dist, color='green') plt.show()
效果展示:
代码展示:
#定义数据扩充 train_datagen = ImageDataGenerator( rescale=1.0/255, zoom_range= 0.2, horizontal_flip=True, shear_range=0.2, validation_split=0.2) test_datagen = ImageDataGenerator(rescale=1.0/255) #将图像读取到数据扩充 #生成批量扩增数据 train = train_datagen.flow_from_directory( train_path, subset='training', color_mode = 'grayscale', target_size = (48, 48), batch_size = 64, shuffle=True, class_mode = 'categorical') valid = train_datagen.flow_from_directory( train_path, subset='validation', color_mode = 'grayscale', target_size = (48, 48), batch_size = 64, shuffle=True, class_mode = 'categorical') test = test_datagen.flow_from_directory( test_path, color_mode = 'grayscale', target_size = (48, 48), batch_size = 64, shuffle=False, class_mode = 'categorical')
效果展示:
代码展示:
train.class_indices
效果展示:
代码展示:
#初始化我们的模型 model = tf.keras.Sequential() #输入层 model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(48, 48, 1), padding='same')) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) #减少维度的最大值池化 model.add(Dropout(0.25)) #test model.add(Conv2D(128, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(128, (3, 3), activation='relu')) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(1024, activation = 'relu')) model.add(Dropout(0.5)) model.add(Dense(7, activation = 'softmax')) model.summary()
效果展示:
代码展示:
from sklearn.utils import class_weight class_weights = class_weight.compute_class_weight( class_weight='balanced', classes=np.unique(train.classes), y=train.classes) class_weights = dict(zip(np.unique(train.classes),class_weights)) filepath = 'my_best_model.epoch{epoch:02d}-loss{val_loss:.2f}.hdf5' #使用自定义名称将最佳模型保存到此位置 callbacks = [tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2,patience=5, min_delta=0.0001, verbose=1), tf.keras.callbacks.ModelCheckpoint(filepath=filepath, monitor='val_loss',verbose=1, save_best_only=True, mode='min'), tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1,restore_best_weights=True)] model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001), loss='categorical_crossentropy', metrics=['accuracy']) emotion = model.fit(train, validation_data=valid, epochs=50, callbacks=callbacks, class_weight=class_weights)
效果展示:
代码展示:
loss,acc = model.evaluate(test,verbose=2)
效果展示:
代码展示:
#保存模型/权重 model.save('emotion-detection.h5') model.save_weights('emotion_weight.h5') np.save("history", emotion.history) print('保存成功!.')
代码展示:
plt.figure(figsize=(20,10)) plt.suptitle('Loss & Accuracy Over Time', fontsize=25) plt.subplot(1, 2, 1) plt.xlabel('Epoch', fontsize=18) plt.ylabel('Loss', fontsize=18) plt.plot(emotion.history['loss'], label='Training Loss') plt.plot(emotion.history['val_loss'], label='Validation Loss') plt.legend(loc='upper right') plt.subplot(1, 2, 2) plt.xlabel('Epoch', fontsize=18) plt.ylabel('Accuracy', fontsize=16) plt.plot(emotion.history['accuracy'], label='Training Accuracy') plt.plot(emotion.history['val_accuracy'], label='Validation Accuracy') plt.legend(loc='upper left') plt.show()
效果展示:
代码展示:
train_loss, train_acc = model.evaluate(train) test_loss, test_acc = model.evaluate(test)
效果展示:
代码展示:
mod = load_model('my_best_model.epoch28-loss1.11.hdf5') y_pred = model.predict(train) y_pred = np.argmax(y_pred, axis=1) class_labels = test.class_indices class_labels = {v:k for k,v in class_labels.items()} print('Confusion Matrix') print(confusion_matrix) print('Classification Report') target_names = list(class_labels.values()) print(classification_report(train.classes, y_pred, target_names=target_names))
效果展示:
代码展示:
img = image.load_img(test_path+"/surprised/im30.png",target_size = (48,48),color_mode = "grayscale") img = np.array(img) plt.imshow(img)
效果展示:
代码展示:
labels = sorted(labels) img = np.expand_dims(img,axis = 0) #reshapes to 1,48,48 img = img.reshape(1,48,48,1) img=tf.cast(img,tf.float64) result = model.predict(img) #用模型预测图像的情感 result = list(result[0]) print(result) print(train.class_indices)
效果展示:
代码展示:
img_index = result.index(max(result)) print('Prediction:',labels[img_index])
效果展示:
惊讶测试:
#惊讶的脸 img = image.load_img(test_path+"/fearful/im30.png", target_size=(48,48),color_mode='grayscale') img = np.array(img) plt.imshow(img)
效果展示:
代码展示:
img = np.expand_dims(img,axis = 0) #reshapes to 1,48,48 img = img.reshape(1,48,48,1) img=tf.cast(img,tf.float32) result = model.predict(img) #用模型预测图像的情感 result = list(result[0]) img_index = result.index(max(result)) print('Prediction:',labels[img_index]) plt.figure(figsize=(10,5)) plt.suptitle('Emotion Prediction') ax1 = plt.subplot() plt.xlabel('Emotion', fontsize=14) plt.ylabel('Certainty', fontsize=14) plt.bar(labels, np.array(result), color='blue')
效果展示:
代码展示:
img = image.load_img('E:/demo.png', target_size=(48,48),color_mode='grayscale') img = np.array(img) plt.imshow(img)
效果展示:
代码展示:
img = np.expand_dims(img,axis = 0) #reshapes to 1,48,48 img = img.reshape(1,48,48,1) img=tf.cast(img,tf.float64) result = model.predict(img) #用模型预测图像的情感 result = list(result[0]) img_index = result.index(max(result)) print('Prediction:',labels[img_index]) plt.figure(figsize=(10,5)) plt.suptitle('Emotion Prediction') ax1 = plt.subplot() plt.xlabel('Emotion', fontsize=14) plt.ylabel('Certainty', fontsize=14) plt.bar(labels, np.array(result), color='blue')
效果展示:
四、总结
机器学习的应用领域非常广泛,学习难度也比较大,说白了就是对于给定的数据进行处理,利用计算机的能力来输出一个output,判断是否符合自己想要的结果,然后不断的优化、完善,最终形成自己的项目。
在做这个项目的时候还是遇到了许多问题,起先就是需要的库版本不对应,在通过百度、博客等方法后成功解决了。然后在项目实施的过程中,还存在着输出结果还不能符合自己的需要,后期通过增加训练方法,优化图像处理方面的能力。
数据集:
https://www.heywhale.com/mw/dataset/639b611e6d0ae1910070d39d/file
完整代码:
#需要的库 import os os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE" import PIL import cv2 import glob import pathlib import zipfile import numpy as np from fastai import * from PIL import Image import tensorflow as tf from fastai.vision import * from tensorflow import keras from tensorflow.keras import layers from tensorflow.keras.layers import * from matplotlib import pyplot as plt from keras.preprocessing import image from keras.preprocessing.image import * from tensorflow.keras.models import Sequential from fastai.metrics import error_rate, accuracy from keras.preprocessing.image import ImageDataGenerator from torch.utils.data import Dataset, DataLoader from keras.callbacks import ModelCheckpoint from sklearn.utils import class_weight from tensorflow.keras.models import load_model from sklearn.metrics import classification_report, confusion_matrix data_dir = 'E:/face/' train_path = data_dir + 'train/' test_path = data_dir + 'test/' def display_images(emotion): plt.figure(figsize=(10,10)) for i in range(1, 10, 1): plt.subplot(3,3,i) #为每个标签形成3x3网格 #img = image.load_img(train_path + '/' + emotion + "/" + os.listdir(train_path + '/' + emotion)[i], target_size=(48, 48)) #load images img = image.load_img(train_path + '/' + emotion + "/" + os.listdir(train_path + '/' + emotion)[i], target_size=(48, 48)) #load images plt.title(emotion) #显示图像标签 plt.imshow(img) #显示图像 plt.tight_layout() #格式化图像 plt.show() for i in os.listdir(train_path): display_images(i) def mylistdir(directory): filelist = os.listdir(directory) return [x for x in filelist if not (x.startswith('.'))] #显示类标签 labels = os.listdir(train_path) print("七种情绪:") labels file_count = len(list(pathlib.Path(data_dir).glob('*/*/*.png'))) print('Total image count:', file_count) file_count = len(list(pathlib.Path(train_path).glob('*/*.png'))) print('Total training images:', file_count) file_count = len(list(pathlib.Path(test_path).glob('*/*.png'))) print('Total testing images:', file_count) #显示7个情绪标签和每个情绪的示例图像 fig, axes = plt.subplots(1, 7, figsize=(20,20)) for i in range(7): ex_image = train_path + labels[i]+ '/' + mylistdir(train_path + labels[i]+'/')[0] axes[i].imshow(plt.imread(ex_image)) axes[i].set_title(labels[i]) plt.show() #打印图像标签分发 for i in labels: print(i, '\nTrain: ' + str(len(os.listdir(train_path + i +'/')))+ ' images' +'\nTest: ' + str(len(os.listdir(test_path+i+'/')))+' images\n') train_dist = np.array([len(os.listdir(train_path+i+'/')) for i in labels]) test_dist = np.array([len(os.listdir(test_path+i+'/')) for i in labels]) x = labels plt.figure(figsize=(20,10)) plt.suptitle('Emotion Distribution', fontsize=25) ax1 = plt.subplot(1,2,1) ax1.set_title('Training Set') plt.xlabel('Emotion Class', fontsize=14) plt.ylabel('Number of Images', fontsize=14) plt.bar(x, train_dist, color='blue') ax2 = plt.subplot(1, 2, 2) ax2.set_title('Test Set') plt.xlabel('Emotion Class', fontsize=14) plt.ylabel('Number of Images', fontsize=14) plt.bar(x, test_dist, color='green') plt.show() #定义数据扩充 train_datagen = ImageDataGenerator( rescale=1.0/255, zoom_range= 0.2, horizontal_flip=True, shear_range=0.2, validation_split=0.2) test_datagen = ImageDataGenerator(rescale=1.0/255) #将图像读取到数据扩充 #生成批量扩增数据 train = train_datagen.flow_from_directory( train_path, subset='training', color_mode = 'grayscale', target_size = (48, 48), batch_size = 64, shuffle=True, class_mode = 'categorical') valid = train_datagen.flow_from_directory( train_path, subset='validation', color_mode = 'grayscale', target_size = (48, 48), batch_size = 64, shuffle=True, class_mode = 'categorical') test = test_datagen.flow_from_directory( test_path, color_mode = 'grayscale', target_size = (48, 48), batch_size = 64, shuffle=False, class_mode = 'categorical') train.class_indices #初始化我们的模型 model = tf.keras.Sequential() #输入层 model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(48, 48, 1), padding='same')) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) #减少维度的最大值池化 model.add(Dropout(0.25)) #test model.add(Conv2D(128, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(128, (3, 3), activation='relu')) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(1024, activation = 'relu')) model.add(Dropout(0.5)) model.add(Dense(7, activation = 'softmax')) model.summary() class_weights = class_weight.compute_class_weight( class_weight='balanced', classes=np.unique(train.classes), y=train.classes) class_weights = dict(zip(np.unique(train.classes),class_weights)) filepath = 'my_best_model.epoch{epoch:02d}-loss{val_loss:.2f}.hdf5' #使用自定义名称将最佳模型保存到此位置 callbacks = [tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2,patience=5, min_delta=0.0001, verbose=1), tf.keras.callbacks.ModelCheckpoint(filepath=filepath, monitor='val_loss',verbose=1, save_best_only=True, mode='min'), tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1,restore_best_weights=True)] model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001), loss='categorical_crossentropy', metrics=['accuracy']) emotion = model.fit(train, validation_data=valid, epochs=50, callbacks=callbacks, class_weight=class_weights) loss,acc = model.evaluate(test,verbose=2) #保存模型/权重 model.save('emotion-detection.h5') model.save_weights('emotion_weight.h5') np.save("history", emotion.history) print('保存成功!.') plt.figure(figsize=(20,10)) plt.suptitle('Loss & Accuracy Over Time', fontsize=25) plt.subplot(1, 2, 1) plt.xlabel('Epoch', fontsize=18) plt.ylabel('Loss', fontsize=18) plt.plot(emotion.history['loss'], label='Training Loss') plt.plot(emotion.history['val_loss'], label='Validation Loss') plt.legend(loc='upper right') plt.subplot(1, 2, 2) plt.xlabel('Epoch', fontsize=18) plt.ylabel('Accuracy', fontsize=16) plt.plot(emotion.history['accuracy'], label='Training Accuracy') plt.plot(emotion.history['val_accuracy'], label='Validation Accuracy') plt.legend(loc='upper left') plt.show() train_loss, train_acc = model.evaluate(train) test_loss, test_acc = model.evaluate(test) mod = load_model('my_best_model.epoch28-loss1.11.hdf5') y_pred = model.predict(train) y_pred = np.argmax(y_pred, axis=1) class_labels = test.class_indices class_labels = {v:k for k,v in class_labels.items()} print('Confusion Matrix') print(confusion_matrix) print('Classification Report') target_names = list(class_labels.values()) print(classification_report(train.classes, y_pred, target_names=target_names)) img = image.load_img(test_path+"/surprised/im30.png",target_size = (48,48),color_mode = "grayscale") img = np.array(img) plt.imshow(img) labels = sorted(labels) img = np.expand_dims(img,axis = 0) #reshapes to 1,48,48 img = img.reshape(1,48,48,1) img=tf.cast(img,tf.float64) result = model.predict(img) #用模型预测图像的情感 result = list(result[0]) print(result) print(train.class_indices) img_index = result.index(max(result)) print('Prediction:',labels[img_index]) #惊讶的脸 img = image.load_img(test_path+"/fearful/im30.png", target_size=(48,48),color_mode='grayscale') img = np.array(img) plt.imshow(img) img = np.expand_dims(img,axis = 0) #reshapes to 1,48,48 img = img.reshape(1,48,48,1) img=tf.cast(img,tf.float32) result = model.predict(img) #用模型预测图像的情感 result = list(result[0]) img_index = result.index(max(result)) print('Prediction:',labels[img_index]) plt.figure(figsize=(10,5)) plt.suptitle('Emotion Prediction') ax1 = plt.subplot() plt.xlabel('Emotion', fontsize=14) plt.ylabel('Certainty', fontsize=14) plt.bar(labels, np.array(result), color='blue') img = image.load_img('E:/demo.png', target_size=(48,48),color_mode='grayscale') img = np.array(img) plt.imshow(img) img = np.expand_dims(img,axis = 0) #reshapes to 1,48,48 img = img.reshape(1,48,48,1) img=tf.cast(img,tf.float64) result = model.predict(img) #用模型预测图像的情感 result = list(result[0]) img_index = result.index(max(result)) print('Prediction:',labels[img_index]) plt.figure(figsize=(10,5)) plt.suptitle('Emotion Prediction') ax1 = plt.subplot() plt.xlabel('Emotion', fontsize=14) plt.ylabel('Certainty', fontsize=14) plt.bar(labels, np.array(result), color='blue')
标签:plt,机器,48,img,情绪,train,图像,import,model From: https://www.cnblogs.com/zhangbiqia2103840118/p/16987250.html