Bachelor of Computer Science (Honours)
Faculty of Information and Communication Technology (Kampar Campus), UTAR
A-12
emotion='Call AWS : '+face_emotion aws+=1
cv2.putText(img=resize, text=emotion, org=(x+15, y-15), fontFace=cv2.FONT_HERSHEY_TRIPLEX, fontScale=0.8, color=(0, 255, 0),thickness=1)
print(emotion)
tempdict['Emotion'].append(emotion)
tempdf = pd.DataFrame.from_dict(tempdict) box=pd.concat([box,tempdf], ignore_index=True) del tempdict, tempdf
else:
if len(box)>0:
for i in range(len(box)):
cv2.rectangle(resize, (box.iloc[i]['x'], box.iloc[i]['y']), (box.iloc[i]['x']+box.iloc[i]['w'],
box.iloc[i]['y']+box.iloc[i]['h']), (0, 0, 255), 5)
cv2.putText(img=resize, text=box.iloc[i]['Emotion'],
org=(x+15, y-15), fontFace=cv2.FONT_HERSHEY_TRIPLEX, fontScale=0.8, color=(0, 255, 0),thickness=1)
cv2.imshow('Teacher-Student Learning Model',resize) key = cv2.waitKey(1)
if key == ord('q'):
break
if key == ord('p'):
cv2.waitKey(-1) #wait until any key is pressed cap.release()
cv2.destroyAllWindows()
print("Number of AWS request Reduced:"+str(local)+" from
"+str(local+aws)+"("+str(local/(local+aws))+")")
A.3 Python Codes of Teacher-Student Machine Learning Model
Bachelor of Computer Science (Honours)
Faculty of Information and Communication Technology (Kampar Campus), UTAR
A-13
TEST_R = 0.1
DATA_DIR_PATH = "same_person"
OUTPUT_DIR = "processed_data"
splitfolders.ratio(DATA_DIR_PATH, OUTPUT_DIR, seed=SEED, ratio=(TRAIN_R, VAL_R, TEST_R))
train_data_dir = f"{OUTPUT_DIR}/train"
valid_data_dir = f"{OUTPUT_DIR}/val"
test_data_dir = f"{OUTPUT_DIR}/test"
return train_data_dir, valid_data_dir
Augmenting the training and validation data
def test_valid_data():
from tensorflow.keras.applications.resnet50 import preprocess_input
from tensorflow.keras.preprocessing.image import ImageDataGenerator
train_data_dir, valid_data_dir=data_preparation() IMG_HEIGHT, IMG_WIDTH = (224, 224)
BATCH_SIZE = 32
train_datagen = ImageDataGenerator(
preprocessing_function=preprocess_input, shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(IMG_HEIGHT, IMG_WIDTH), batch_size=BATCH_SIZE,
class_mode="categorical")
valid_generator = train_datagen.flow_from_directory(
valid_data_dir,
target_size=(IMG_HEIGHT, IMG_WIDTH), batch_size=BATCH_SIZE,
class_mode="categorical")
return train_generator, valid_generator
Training the model
def train_model():
from tensorflow.keras.layers import Dense, GlobalAveragePooling2D
Bachelor of Computer Science (Honours)
Faculty of Information and Communication Technology (Kampar Campus), UTAR
A-14
from tensorflow.keras.applications.resnet50 import ResNet50 from tensorflow.keras.models import Model
train_generator,valid_generator=test_valid_data() EPOCHS = 30
base_model = ResNet50(include_top=False, weights="imagenet") x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(1024, activation="relu")(x)
predictions = Dense(train_generator.num_classes, activation="softmax")(x)
model = Model(inputs=base_model.input, outputs=predictions) for layer in base_model.layers:
layer.trainable = False
model.compile(
optimizer="adam",
loss="categorical_crossentropy", metrics=["accuracy"])
history = model.fit(train_generator,
validation_data=valid_generator, epochs=EPOCHS)
model.save('Saved_Model\ResNet50_FER.h5') return history
Evaluating the model using testing data
import tensorflow as tf
from tensorflow.keras.applications.resnet50 import preprocess_input from tensorflow.keras.preprocessing.image import ImageDataGenerator model=tf.keras.models.load_model('Saved_Model\ResNet50_FER.h5') test_datagen =
ImageDataGenerator(preprocessing_function=preprocess_input) IMG_HEIGHT, IMG_WIDTH = (224, 224)
test_generator = test_datagen.flow_from_directory(
f"processed_data/test",
target_size=(IMG_HEIGHT, IMG_WIDTH), batch_size=1,
class_mode="categorical")
test_loss, test_acc =model.evaluate(test_generator, verbose=2) print('\nTest Accuracy:', test_acc)
Predicting images using the model
Bachelor of Computer Science (Honours)
Faculty of Information and Communication Technology (Kampar Campus), UTAR
A-15
import pandas as pd import numpy as np
IMG_HEIGHT, IMG_WIDTH = (224, 224)
from keras.preprocessing.image import load_img, img_to_array def preprocess_image(path):
img_width=IMG_HEIGHT; img_height=IMG_HEIGHT
img = load_img(path, target_size = (img_height, img_width)) a = img_to_array(img)
a = np.expand_dims(a, axis = 0) return a
import tensorflow as tf filename="test/happy.png"
classifier=tf.keras.models.load_model('Saved_Model\ResNet50_FER.h5') array = classifier.predict(preprocess_image(filename), batch_size=1, verbose=1)
answer = np.argmax(array, axis=1) print(answer)
emotion_label=['ANGRY', 'CALM', 'CONFUSED', 'DISGUSTED', 'FEAR', 'HAPPY', 'SAD', 'SURPRISED']
print(emotion_label[answer[0]]) print(array[0][answer[0]])
Predicting the video frame using the model with the teacher-student learning approach
import numpy as np import cv2
import tensorflow as tf import pandas as pd
from datetime import datetime face_cascade =
cv2.CascadeClassifier('C:/Users/User/Documents/Study/FYP/Python evaluation/opencv-
master/data/haarcascades/haarcascade_frontalface_alt2.xml') def rescale_frame(frame, percent=75):
width = int(frame.shape[1] * percent/ 100) height = int(frame.shape[0] * percent/ 100) dim = (width, height)
return cv2.resize(frame, dim, interpolation =cv2.INTER_AREA)
# Before reading the video input
classifier=tf.keras.models.load_model('Saved_Model\ResNet50_FER.h5') font = cv2.FONT_HERSHEY_SIMPLEX
fontScale = 10
fontColor = (0,0,255) thickness = 10
lineType = 2
Bachelor of Computer Science (Honours)
Faculty of Information and Communication Technology (Kampar Campus), UTAR
A-16
cap = cv2.VideoCapture('video/sample2.mp4')
emotion_label=['ANGRY', 'CALM', 'CONFUSED', 'DISGUSTED', 'FEAR', 'HAPPY', 'SAD', 'SURPRISED']
import boto3
weimun19_1utar_rekognition =
boto3.client('rekognition',aws_access_key_id='AKIAWKKRIJIYJXQKJTNU', aws_secret_access_key='b71RuxYbiIcl4WcAVoicHNbwDTM5pZI5w kzzR17i',
region_name='ap-southeast-1') aws=0
local=0
FPS=int(cap.get(cv2.CAP_PROP_FPS)) print("FPS: "+str(FPS))
# Using cv2.putText() method
IMG_HEIGHT, IMG_WIDTH = (224, 224) cur_frame=0
box = pd.DataFrame(columns=['x','y','w','h','Emotion']) while(True):
ret, frame = cap.read() if not ret:
break cur_frame+=1
resize = rescale_frame(frame, percent=25) if cur_frame % FPS == 0:
box.drop(box.index, inplace=True)
gray = cv2.cvtColor(resize, cv2.COLOR_BGR2GRAY) faces = face_cascade.detectMultiScale(gray, 1.1, 4) for (x, y, w, h) in faces:
tempdict={'x':[],'y':[],'w':[],'h':[],'Emotion':[]}
tempdict['x'].append(x) tempdict['y'].append(y) tempdict['w'].append(w) tempdict['h'].append(h)
cropped = resize[y:y + h, x:x + w]
cv2.rectangle(resize, (x, y), (x+w, y+h), (0, 0, 255), 5) resized=cv2.resize(cropped, (IMG_HEIGHT, IMG_WIDTH)) print(type(resized))
a = np.expand_dims(resized, axis = 0)
array = classifier.predict(a, batch_size=1, verbose=1) answer = np.argmax(array, axis=1)
if(array[0][answer[0]]<0.8):
ret, buf = cv2.imencode('.jpg', cropped)
Bachelor of Computer Science (Honours)
Faculty of Information and Communication Technology (Kampar Campus), UTAR
A-17
rekognition_response =
weimun19_1utar_rekognition.detect_faces(Image={'Bytes':buf.tobytes()}
, Attributes=['ALL'])
#print(rekognition_response)
results=rekognition_response.get('FaceDetails') emotion='Call AWS : Not Detected'
for result in results:
face_emotion_confidence = 0 face_emotion = None
for emotion_result in result.get('Emotions'):
if emotion_result.get('Confidence') >=
face_emotion_confidence:
face_emotion_confidence = emotion_result['Confidence']
face_emotion = emotion_result.get('Type') print(face_emotion)
emotion='Call AWS : '+face_emotion
#cv2.imwrite("same_person/"+face_emotion+"/"+str(datetime.n ow().strftime("%m%d%H%M%S"))+".jpg",cropped)
cv2.putText(img=resize, text=emotion, org=(x+15, y-15), fontFace=cv2.FONT_HERSHEY_TRIPLEX, fontScale=0.8, color=(0, 255, 0),thickness=1)
aws+=1 else:
local+=1
emotion="Model: "+emotion_label[answer[0]]
print(emotion+" "+str(array[0][answer[0]]))
cv2.putText(img=resize, text=emotion, org=(x+15, y-15), fontFace=cv2.FONT_HERSHEY_TRIPLEX, fontScale=0.8, color=(0, 255, 0),thickness=1)
tempdict['Emotion'].append(emotion)
tempdf = pd.DataFrame.from_dict(tempdict) box=pd.concat([box,tempdf], ignore_index=True) del tempdict, tempdf
else:
if len(box)>0:
for i in range(len(box)):
cv2.rectangle(resize, (box.iloc[i]['x'], box.iloc[i]['y']), (box.iloc[i]['x']+box.iloc[i]['w'],
box.iloc[i]['y']+box.iloc[i]['h']), (0, 0, 255), 5)
cv2.putText(img=resize, text=box.iloc[i]['Emotion'],
org=(x+15, y-15), fontFace=cv2.FONT_HERSHEY_TRIPLEX, fontScale=0.8, color=(0, 255, 0),thickness=1)
cv2.imshow('Teacher-Student Learning Model',resize) key = cv2.waitKey(1)
if key == ord('q'):
break
Bachelor of Computer Science (Honours)
Faculty of Information and Communication Technology (Kampar Campus), UTAR
A-18
if key == ord('p'):
cv2.waitKey(-1) #wait until any key is pressed cap.release()
cv2.destroyAllWindows()
print("Number of AWS request Reduced:"+str(local)+" from
"+str(local+aws)+"("+str(local/(local+aws))+")")
#after the recognition process, retrain again print("Start training again")
train_model()
A.4 Dataset Used in Down Sampling and Spatial Trimming