How to use pytorch on a single node?

assigned to @px3192

By till.riedel on 2022-05-18T10:47:17 (imported from GitLab)

Package Installation

Pytorch

For conda user:

!conda install pytorch==1.10.0 torchvision==0.11.0 torchaudio==0.10.0 -c pytorch or

For pip user:

!pip3 install torch torchvision torchaudio

For other choice (e.g., with/without GPU, different OS), please see https://pytorch.org/.

Note that to use SDIL GPU, a higher version must be installed, whatever conda or pip

pip3 install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
Packages for GPU-checking

For conda user:

!conda install -c conda-forge psutil

!conda install -c conda-forge humanize

!conda install -c conda-forge gputil

For pip user:

!pip3 psutil

!pip3 install humanize

!pip3 install GPUtil
Packages for others

For conda user:

!conda install calender

For pip user:

!pip3 install calender

Import packages

# for pytorch
import torch
import torchvision
# for GPU checking
import psutil
import humanize
import os
import GPUtil as GPU
import matplotlib.pyplot as plt
# for training NN
import numpy as np
import calendar
import time
from datetime import datetime

Settings

# set a random seed for the reproducibility
torch.manual_seed(0);

# GPU check
GPUs = GPU.getGPUs()
# XXX: only one GPU on Colab and isn’t guaranteed
gpu = GPUs[0]
def printm():
    process = psutil.Process(os.getpid())
    print("Gen RAM Free: " + humanize.naturalsize( psutil.virtual_memory().available ), " | Proc size: " + humanize.naturalsize( process.memory_info().rss))
    print("GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))
printm()

Gen RAM Free: 387.6 GB  | Proc size: 1.1 GB
GPU RAM Free: 40314MB | Used: 39MB | Util   0% | Total 40960MB

# select GPU if available, otherwise, CPU
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
device

Prepare Dataset

Define Data-Preprocessing

We apply easily the torchvision.transform to do the pre-processing. For further information, see https://pytorch.org/vision/stable/transforms.html.

transform = torchvision.transforms.Compose([torchvision.transforms.ToTensor(),
                                            torchvision.transforms.Normalize([0.5], [0.5])])

Download dataset

create a folder to store data
download data. If the dataset has been already downloaded once, download could be False, staying True is also no problem.
apply the transform on dataset

Note that, their might be a warning about "non-writable NumPy Arrray". To solve this warning, please go to the path for conda:

Anaconda (or Miniconda)/envs/MyEnvironment/lib/python3.8/site-packages/torchvision/datasets/mnist.py

in line 498 (might changes slightly from version to version), delete copy=False, i.e., from

return torch.from_numpy(parsed.astype(m[2]), copy=False).view(*s) to

return torch.from_numpy(parsed.astype(m[2])).view(*s).

if not os.path.exists('./data'):
    os.mkdir('./data')

train_data_origin = torchvision.datasets.FashionMNIST(root='./data/', train=True, download=True, transform=transform)
train_data_origin

# split train data into train and valid data
N_train = len(train_data_origin)
N_train_new = int(N_train * 0.8)
N_valid_new = N_train - N_train_new
N_train_new, N_valid_new

train_data, valid_data = torch.utils.data.random_split(train_data_origin, [N_train_new, N_valid_new])
len(train_data), len(valid_data)

test_data = torchvision.datasets.FashionMNIST(root='./data/', train=False, download=True, transform=transform)
test_data

Build dataloader

Details for loader see https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader.

train_loader = torch.utils.data.DataLoader(train_data, batch_size=64, shuffle=True)
valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=64, shuffle=True)
test_loader  = torch.utils.data.DataLoader(test_data,  batch_size=64, shuffle=True)

Visualize data

data_temp = iter(train_loader)
img_temp, label_temp = data_temp.next()

fig = plt.figure(figsize=(15,5))
for i in range(20):
    ax = fig.add_subplot(5, 10, i+1, xticks=[], yticks=[])
    ax.imshow(np.squeeze(img_temp[i]), cmap='gray')
    ax.set_title(label_temp[i].item())
    fig.tight_layout()

Training Neural Networks

Define Neural Networks

Check in-/output dimensionality

img_temp[0].shape

N_feature = np.prod(img_temp[0].shape)
N_feature

N_class = 10

model = torch.nn.Sequential(torch.nn.Linear(N_feature, 256), torch.nn.PReLU(),
                            torch.nn.Linear(256, 128),       torch.nn.PReLU(),
                            torch.nn.Linear(128, 64),        torch.nn.PReLU(),
                            torch.nn.Linear(64, N_class),    torch.nn.LogSoftmax(dim=1))
model.to(device)

Define loss function and optimizer

# Define the loss
Lossfunction = torch.nn.CrossEntropyLoss()
Lossfunction.to(device)

# Define the optimizer
Optimizer = torch.optim.Adam(model.parameters(), lr = 1e-4)

Define the training process

def train_nn(nn, train_loader, valid_loader, optimizer, lossfunction, Epoch=10**10):
    # dir for save temporary files
    if not os.path.exists('./temp'):
        os.mkdir('./temp')
    
    # create an unique ID for saving temp file, avoiding file overwriting while multiple training
    training_ID = ts = int(calendar.timegm(time.gmtime()))
    print(f'The ID for this training is {training_ID}.')
    
    # initialize best valid loss for saving the best model
    best_valid_loss = 10 ** 10
    
    # arrays to save training process
    train_losses = []
    valid_losses = []
    train_accs = []
    valid_accs = []
    
    # to count the epoch without any improvement, for early stop
    not_imporved = 0
    
    # training
    for epoch in range(Epoch):
        # timer
        start_time = datetime.now()
        
        # some temp variables to calculate the loss and acc from mini-batch to batch
        num_of_mini_batch  = []
        loss_of_mini_batch = []
        acc_of_mini_batch  = []
        for X_train, y_train in train_loader:
            # reshape 2D images into 1D, also transfer to device
            X_train = X_train.view(X_train.shape[0], -1).to(device)
            y_train = y_train.to(device)
            
            # forward propagation
            prediction_train = nn(X_train)
            
            # calculate loss
            train_loss_mini_batch = lossfunction(prediction_train, y_train)

            # calculate predicted class of input data
            yhat_train = torch.argmax(prediction_train.data, 1)
            
            # calculate how many predictions are correct
            train_correct = torch.sum(yhat_train == y_train.data)
            
            # calculate accuracy of prediction
            train_acc_mini_batch = train_correct / y_train.numel()

            # update parameters in model
            optimizer.zero_grad()
            train_loss_mini_batch.backward()
            optimizer.step()

            # loss and acc from mini-batch
            num_of_mini_batch.append(X_train.shape[0])
            loss_of_mini_batch.append(train_loss_mini_batch.item())
            acc_of_mini_batch.append(train_acc_mini_batch.item())

        # convert and record loss/acc from mini-batch to batch
        train_loss = np.average(loss_of_mini_batch, weights=num_of_mini_batch)
        train_losses.append(train_loss)
        train_acc = np.average(acc_of_mini_batch, weights=num_of_mini_batch)
        train_accs.append(train_acc)

        # similar as training, calculate loss and accuracy on valid data
        num_of_mini_batch  = []
        loss_of_mini_batch = []
        acc_of_mini_batch  = []
        with torch.no_grad():
            for X_valid, y_valid in valid_loader:
                X_valid = X_valid.view(X_valid.shape[0], -1).to(device)
                y_valid = y_valid.to(device)
                prediction_valid = nn(X_valid) 
                valid_loss_mini_batch = lossfunction(prediction_valid, y_valid).data
                yhat_valid = torch.argmax(prediction_valid.data, 1)
                valid_correct = torch.sum(yhat_valid == y_valid.data)
                valid_acc_mini_batch = valid_correct / y_valid.numel()

                num_of_mini_batch.append(X_valid.shape[0])
                loss_of_mini_batch.append(valid_loss_mini_batch.item())
                acc_of_mini_batch.append(valid_acc_mini_batch.item())

            valid_loss = np.average(loss_of_mini_batch, weights=num_of_mini_batch)
            valid_losses.append(valid_loss)
            valid_acc = np.average(acc_of_mini_batch, weights=num_of_mini_batch)
            valid_accs.append(valid_acc)

        # if valid loss in current epoch is better than previous one, save this model
        if valid_loss < best_valid_loss:
            best_valid_loss = valid_loss
            torch.save(nn, f'./temp/NN_temp_{training_ID}')
            random_state = torch.random.get_rng_state()
            torch.save(random_state, f'./temp/NN_temp_random_state_{training_ID}')
            not_imporved = 0
        # if not, that means in this epoch, the model is not improved.
        else:
            not_imporved += 1
            if not_imporved > 5:
                print('Early stop.')
                break
        
        # timer
        end_time = datetime.now()
        
        # print information about current epoch
        if epoch % 1 == 0:
            print(f'| Epoch: {epoch:-5d} | Train acc: {train_acc:-.5f} | Train loss: {train_loss:-.5e} | Valid acc: {valid_acc:-.5f} | Valid loss: {valid_loss:-.5e} | run time: {end_time-start_time} |')
            
    print('Finished.')
    
    return torch.load(f'./temp/NN_temp_{training_ID}'), train_losses, valid_losses, train_accs, valid_accs

model, train_loss, valid_loss, train_acc, valid_acc = train_nn(model, train_loader, valid_loader, Optimizer, Lossfunction)

plt.plot(train_loss, label='train loss');
plt.plot(valid_loss, label='valid loss');
plt.xlabel('epoch')
plt.xlabel('loss')
plt.title('Loss of training')
plt.legend();

plt.plot(train_acc, label='train acc');
plt.plot(valid_acc, label='valid acc');
plt.xlabel('epoch')
plt.xlabel('accuracy')
plt.title('Acc of training')
plt.legend();

num_of_mini_batch  = []
loss_of_mini_batch = []
acc_of_mini_batch  = []
with torch.no_grad():
    for X_test, y_test in test_loader:
        X_test = X_test.view(X_test.shape[0], -1).to(device)
        y_test = y_test.to(device)
        prediction_test = model(X_test) 
        test_loss_mini_batch = Lossfunction(prediction_test, y_test).data
        yhat_test = torch.argmax(prediction_test.data, 1)
        test_correct = torch.sum(yhat_test == y_test.data)
        test_acc_mini_batch = test_correct / y_test.numel()

        num_of_mini_batch.append(X_test.shape[0])
        loss_of_mini_batch.append(test_loss_mini_batch.item())
        acc_of_mini_batch.append(test_acc_mini_batch.item())

    test_loss = np.average(loss_of_mini_batch, weights=num_of_mini_batch)
    test_acc = np.average(acc_of_mini_batch, weights=num_of_mini_batch)

test_acc, test_loss

Save trained model

if not os.path.exists('./result'):
        os.mkdir('./result')
torch.save(model, './result/MyModel.nn')

By haibin.zhao on 2022-05-25T15:11:35 (imported from GitLab)

added answered label

added langpython platformJupyter rookie woes labels

unassigned @till.riedel