ML Batching from Scratch

May 9, 2023

For this article, we are going to batch from scratch. Previously, we created a 2-layer CNN, added activation functions, the adam optimizer, and initialized random values by addressing variance. Now, we can put it all together to create a model that is usable. In the prior articles, we updated the weights after every iteration over the training data. Batching allows us to update the weights after averaging the derivative of the loss with respect to a given weight over the entire batch. This can be beneficial because it doesn't allow bad training data to have a sizeable impact on the weights. Averaging allows the weights to be steered in the direction of an entire batch, not just one sample. 

An additional benefit is that it allows the model to train over the same data. You can iterate different combinations of batches over the same dataset. Also, this is great for parallel computing as the derivatives for an entire batch can be done independently of other elements in the batch. See commented code below:

import numpy as np

mnistlocation = “INSERT MNIST FILE LOCATION” #you can download the file here

# For example: “/Users/enrichmentcap/Downloads/mndata.npz”

trains, tests = 60000, 10000 #depends on computer processing speed. If computer isn't fast, reduce size of training and test dataset

trainimages, trainlabels, testimages, testlabels = np.load(mnistlocation)['trainimages'][:trains] / 255, np.load(mnistlocation)['trainlabels'][:trains], np.load(mnistlocation)['testimages'][:tests] / 255, np.load(mnistlocation)['testlabels'][:tests]

##from tensorflow.keras.datasets import mnist #uncomment if using tensorflow library to retrieve dataset

##(trainimages, trainlabels), (testimages, testlabels) = mnist.load_data()

##trainimages, testimages = trainimages[:train]/255, testimages[:test]/255

np.random.seed(0)

classes = len(np.unique(trainlabels))

imw = trainimages.shape[2]

imh = trainimages.shape[1]

lr = .001

fsw = 8

fsh = 8

fsw2 = 4

fsh2 = 4

filts =  16

filts2 = 8

step = 1

step2 = 2

rsw = (imw - fsw) // step + 1

rsh = (imh - fsh) // step + 1

rsw2 = (rsw - fsw2) // step2 + 1

rsh2 = (rsh - fsh2) // step2 + 1

kern = np.random.randn(filts, fsh, fsw) * np.sqrt(2 / (fsh*fsw))# + filts*fsh*fsw)) # (2 / (X*Y))**.5 = he normal initializer 

kb = np.zeros(filts)

kern2 = np.random.randn(filts2, filts, fsh2, fsw2) * np.sqrt(2 / (filts*fsh2*fsw2))# + filts2*fsh2*fsw2))

kb2 = np.zeros(filts2)

w = np.random.randn(classes, filts2, rsh2, rsw2) * np.sqrt(6 / (filts2*rsh2*rsw2 + classes)) # (6 / (X*Y + Z))**.5 = glorot normal initializer

b = np.zeros(classes)

#adam optimizer params

b1 = .9

b2 = .999

eps = 1e-7

mw = 0

vw = 0

mb = 0

vb = 0

mk = 0

vk = 0

mk2 = 0

vk2 = 0

mkb = 0

vkb = 0

mkb2 = 0

vkb2 = 0

epochs = 10

bs = 100 #batch size

nbatch = trains // bs #number of batches per epoch

for e in range(epochs): #epochs go through the same dataset, but in different batches; this allows one to train over the same dataset while continuing to learn

    print('epoch: %d'%e)

    np.random.seed(e) 

    ibatch = np.random.permutation(trainimages[:trains]) #training set of images to be batched

    np.random.seed(e) #reverting back to the random seed ensures that train images and labels are shuffled exactly the same, despite being different sizes

    lbatch = np.random.permutation(trainlabels[:trains]) #training set of labels to be batched

    for nb in range(nbatch):

        #these derivatives are backpropagated to trainable weights; need to be reset after every batch

        #trainable weights are only updated once per batch; each batch takes the average of all these values, then backpropagates to the trainable weights

        bdLdw = np.zeros((bs, classes, filts2, rsh2, rsw2)) 

        bdLdb = np.zeros((bs, classes)) 

        bdLdkern2 = np.zeros((bs, filts2, filts, fsh2, fsw2))

        bdLdkb2 = np.zeros((bs, filts2))

        bdLdkern = np.zeros((bs, filts, fsh, fsw))

        bdLdkb = np.zeros((bs, filts))

        for i in range(bs):

            xx, label, label[lbatch[nb*bs + i]] = ibatch[nb*bs + i], np.zeros(classes), 1

            k = np.zeros((filts, rsh, rsw))

            for j in range(rsh):

                for jj in range(rsw):

                    k[:,j,jj] = (kern * xx[step*j:step*j+fsh, step*jj:step*jj+fsw].reshape(1,fsh,fsw)).sum(axis=(1,2)) + kb

            kr = np.maximum(0, k) #relu activation

            kk = np.zeros((filts2, rsh2, rsw2))

            for j0 in range(rsh2):

                for jj0 in range(rsw2):

                    kk[:,j0,jj0] = (kern2 * kr[:,step2*j0:step2*j0+fsh2, step2*jj0:step2*jj0+fsw2].reshape(1,filts,fsh2,fsw2)).sum(axis=(1,2,3)) + kb2 #kk.shape = (filts2, rsize2, rsize2)

            kkr = np.maximum(0, kk) #relu activation

            x = (w * kkr.reshape(1,filts2,rsh2,rsw2)).sum(axis=(1,2,3)) + b

            y = np.exp(x) / np.sum(np.exp(x))

            dydx = -np.exp(x)[lbatch[nb*bs + i]] * np.exp(x) / np.sum(np.exp(x))**2

            dydx[lbatch[nb*bs + i]] = np.exp(x)[lbatch[nb*bs + i]] * (np.sum(np.exp(x)) - np.exp(x)[lbatch[nb*bs + i]]) / np.sum(np.exp(x))**2

            dLdy = 1 / y[lbatch[nb*bs + i]]

            dLdx = dLdy * dydx

            dxdw, dxdb = kkr, 1

            dLdw = dLdx.reshape(classes,1,1,1) * dxdw.reshape(1, filts2, rsh2, rsw2)

            dLdb = dLdx * dxdb

            bdLdw[i] = dLdw

            bdLdb[i] = dLdb

            dxdkkr = w #(classes, filts2*rsize2**2)

            dLdkkr = (dLdx.reshape(classes, 1, 1, 1) * dxdkkr).sum(axis=0)#dLdkk represents all class loss integrated into appropriate positions in kk output; meaning which positions in kk led to more loss or error

            dkkrdkk = np.array(kk > 0, dtype = float) #relu activation derivative

            dLdkk = dLdkkr * dkkrdkk

            dkkdkern2 = kr

            dLdkern2 =  np.zeros((filts2,filts,fsh2,fsw2))

            for f in range(filts2):

                for j000 in range(fsh2):

                    for jj000 in range(fsw2):

                        dLdkern2[f, :, j000, jj000] = (dLdkk[f].reshape(1,rsh2,rsw2) * dkkdkern2[:,j000:j000+rsh2,jj000:jj000+rsw2]).sum(axis=(1,2))

            bdLdkern2[i] = dLdkern2

            dkkdkb2 = 1

            dLdkb2 = dLdkk.sum(axis=(1,2)) * dkkdkb2

            bdLdkb2[i] = dLdkb2

            dkkdkr = kern2 # shape [filts2, filts * fsize2**2]

            dLdkr = np.zeros((filts, rsh, rsw))

            for ooo in range(filts):

                for o in range(rsh2):

                    for oo in range(rsw2):

                        dLdkr[ooo,o*step2:o*step2+fsh2,oo*step2:oo*step2+fsw2] += (dLdkk[:,o,oo].reshape(filts2,1,1) * dkkdkr[:,ooo,:,:]).sum(axis=0) #[filts2,] @ [filts2,fsize2,fsize2]

            dkrdk = np.array(k > 0, dtype = float) #relu activation derivative

            dLdk = dLdkr * dkrdk

            dkdkern = xx

            dLdkern = np.zeros((filts,fsh,fsw))

            for j00 in range(fsh):

                for jj00 in range(fsw):

                    dLdkern[:,j00,jj00] = (dkdkern[j00:j00+rsw, jj00:jj00+rsh].reshape(1,rsw,rsh) * dLdk).sum(axis=(1,2))

            bdLdkern[i] = dLdkern

            dkdkb = 1

            dLdkb = dLdk.sum(axis=(1,2)) * dkdkb

            bdLdkb[i] = dLdkb

        #adam optimizer; only updates weights after entire batch has been iterated through

        lrt = lr * (1 - b2**(e*nbatch + nb + 1))**.5 / (1 - b1**(e*nbatch + nb + 1))        

        mk = b1 * mk + (1 - b1) * bdLdkern.mean(axis=0)

        vk = b2 * vk + (1 - b2) * bdLdkern.mean(axis=0)**2

        kern = kern + lrt * mk / (vk**.5 + eps)

        mkb = b1 * mkb + (1 - b1) * bdLdkb.mean(axis=0)

        vkb = b2 * vkb + (1 - b2) * bdLdkb.mean(axis=0)**2

        kb = kb + lrt * mkb / (vkb**.5 + eps)

        mk2 = b1 * mk2 + (1 - b1) * bdLdkern2.mean(axis=0)

        vk2 = b2 * vk2 + (1 - b2) * bdLdkern2.mean(axis=0)**2

        kern2 = kern2 + lrt * mk2 / (vk2**.5 + eps)

        mkb2 = b1 * mkb2 + (1 - b1) * bdLdkb2.mean(axis=0)

        vkb2 = b2 * vkb2 + (1 - b2) * bdLdkb2.mean(axis=0)**2

        kb2 = kb2 + lrt * mkb2 / (vkb2**.5 + eps)

        mw = b1 * mw + (1 - b1) * bdLdw.mean(axis=0)                                    

        vw = b2 * vw + (1 - b2) * bdLdw.mean(axis=0)**2

        w = w + lrt * mw / (vw**.5 + eps)    

        mb = b1 * mb + (1 - b1) * bdLdb.mean(axis=0)

        vb = b2 * vb + (1 - b2) * bdLdb.mean(axis=0)**2

        b = b + lrt * mb / (vb**.5 + eps)

checke = np.zeros(tests)               

for i in range(tests):

    xx = testimages[i]

    k = np.zeros((filts, rsh, rsw))

    for j in range(rsh):

        for jj in range(rsw):

            k[:,j,jj] = (kern * xx[step*j:step*j+fsh, step*jj:step*jj+fsw].reshape(1,fsh,fsw)).sum(axis=(1,2)) + kb

    kr = np.maximum(0, k)

    kk = np.zeros((filts2, rsh2, rsw2))

    for j0 in range(rsh2):

        for jj0 in range(rsw2):

            kk[:,j0,jj0] = (kern2 * kr[:,step2*j0:step2*j0+fsh2, step2*jj0:step2*jj0+fsw2].reshape(1,filts,fsh2,fsw2)).sum(axis=(1,2,3)) + kb2 #kk.shape = (filts2, rsize2, rsize2)

    kkr = np.maximum(0, kk)

    x = (w * kkr.reshape(1,filts2,rsh2,rsw2)).sum(axis=(1,2,3)) + b

    if testlabels[i] == np.argmax(x):

        checke[i] = 1

print(len(np.flatnonzero(checke==1))/tests)