Confusion Matrix using Sci-kit Learn and Keras randomly shifted over and no longer accurate

I have been re-running the same Keras program over and over for tuning purposes, changing nothing but the number of Epochs, or adding.removing layers. Now no matter what I do the confusion matrix (using the Sci-kit Learn one) is wrong and seems to have shifted over (instead of it being on the diagonal). This seems to have happened overnight, and now when I go back to older versions of the program which initially had a correct confusion matrix, those are no longer accurate either. What could have caused this seemingly random change? None of the files in the program have changed. I am completely stumped and can't seem to find any reason for this.

import os, shutil
import pandas as pd
import numpy as np
import keras 
from keras.models import Model
directory = r'D:\Machine Learning\10-monkey-species'
labels = directory +'\monkey_labels.txt'

columns = ['Label','Latin Name', 'Common Name','Train Images', 'Validation Images']
monkey_labels = pd.read_csv(labels,names=columns, skiprows=1)
print(monkey_labels.info())
print(list(monkey_labels))


from keras import layers, optimizers
from keras.layers import Input, Flatten
from keras.models import Model
from keras.utils import plot_model

inputs = Input(shape=(150,150,3))

# a layer instance is callable on a tensor, and returns a tensor
x = layers.Conv2D(32,(3,3), activation='relu')(inputs)
x = layers.Conv2D(32, (3,3),activation='relu')(x)
x = layers.Conv2D(32, (3,3),activation='relu')(x)
x = layers.MaxPooling2D(2,2)(x)
x = layers.Conv2D(64, (3,3),activation='relu')(x)
x = layers.Conv2D(64, (3,3),activation='relu')(x)
x = layers.MaxPooling2D(2,2)(x)
x = layers.Flatten()(x)
x = layers.Dropout(0.3)(x)
x = layers.Dense(64,activation='relu')(x)

predictions = layers.Dense(10, activation='softmax')(x)


model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer='rmsprop',
              loss='categorical_crossentropy',
              metrics=['accuracy'])


from keras.preprocessing.image import ImageDataGenerator


train_datagen = ImageDataGenerator(rescale=1./255,
                                  rotation_range = 40,
                                  width_shift_range =0.2,
                                  height_shift_range = 0.2,
                                  shear_range = 0.2,
                                   zoom_range = 0.3,
                                   horizontal_flip=True,
                                  fill_mode='nearest')


val_datagen = ImageDataGenerator(rescale=1./255)


batch_size = 64

# Directories to train and validation sets:
train_dir = r'D:\Machine Learning\10-monkey-species\training\training'
val_dir = r'D:\Machine Learning\10-monkey-species\validation\validation'

#Generators:
train_generator = train_datagen.flow_from_directory(
                    train_dir,
                    target_size =(150,150),
                    batch_size=batch_size,
                    class_mode='categorical',
                    seed=1337)
val_generator = val_datagen.flow_from_directory(
                    val_dir,
                    target_size =(150,150),
                    batch_size=batch_size,
                    class_mode='categorical',
                    shuffle=False,
                    seed=1337)

train_num = train_generator.samples
val_num = val_generator.samples 

from keras import callbacks



filepath=str(os.getcwd()+"/best monkey model.h5f")


callbacks = [keras.callbacks.ReduceLROnPlateau(monitor='val_loss',
                                                     factor = 0.2,
                                                     patience = 3,
                                                      cooldown=3),
                    keras.callbacks.ModelCheckpoint(filepath=filepath,
                                                     monitor='val_acc',
                                                    verbose=1,
                                                     save_best_only=True,
                                                    mode='max')]

history = model.fit_generator(train_generator,
                             steps_per_epoch = (train_num // batch_size),
                             epochs=50,
                             validation_data=val_generator,
                              callbacks=callbacks,
                             validation_steps=val_num // batch_size)

import sklearn
from sklearn.metrics import confusion_matrix, classification_report

number_of_test_samples = len(val_generator.classes)


# Get the filenames from the generator
file_names = val_generator.filenames


# Get the ground truth from generator
classes = val_generator.classes


# Get the predictions from the model using the generator
predictions = model.predict_generator(val_generator, steps=number_of_test_samples// batch_size+1,verbose=1)


predicted_classes = np.argmax(predictions,axis=1)

cm = confusion_matrix(classes, predicted_classes)
print(cm)

I get the below result, which looks shifted over:

[[ 0  1 17  7  0  0  1  0  0  0]
 [ 0  0  0 16  5  2  1  1  3  0]
 [ 1  1  0  1 10  5  2  4  1  2]
 [ 4  0  0  1  0 10  8  1  2  4]
 [ 0  3  0  1  2  4 11  4  1  0]
 [ 0  1  0  0  1  1  2 15  8  0]
 [ 1  0  0  1  0  0  1  1 15  7]
 [11  0  0  1  0  2  0  0  0 14]
 [10  4  0  1  1  2  0  4  1  4]