'''Training script. ''' import os from tqdm import tqdm import torch import torch.nn as nn from torch.utils.data import DataLoader from torch.optim import Adam, lr_scheduler from torchsummary import summary from torchvision import transforms import torch.distributed as dist import torch.multiprocessing as mp from models.resnet50 import ResNet50 from runtime_args import args from load_dataset import LoadDataset from plot import plot_loss_acc from helpers import calculate_accuracy device = torch.device("cuda:0" if torch.cuda.is_available() and args.device == 'gpu' else 'cpu') if not os.path.exists(args.graphs_folder) : os.mkdir(args.graphs_folder) model_save_folder = 'resnet_cbam/' if args.use_cbam else 'resnet/' if not os.path.exists(model_save_folder) : os.mkdir(model_save_folder) def train(gpu, args): '''Init models and dataloaders and train/validate model. ''' rank = args.rank * args.gpus + gpu world_size = args.gpus * args.nodes dist.init_process_group(backend='nccl', init_method='env://', world_size=world_size, rank=rank) model = ResNet50(image_depth=args.img_depth, num_classes=args.num_classes, use_cbam=args.use_cbam) torch.cuda.set_device(gpu) model.cuda(gpu) optimizer = Adam(model.parameters(), lr=args.learning_rate) lr_decay = lr_scheduler.ExponentialLR(optimizer, gamma=args.decay_rate) criterion = torch.nn.CrossEntropyLoss().cuda(gpu) summary(model, (3, 224, 224)) model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu]) train_dataset = LoadDataset(dataset_folder_path=args.data_folder, image_size=args.img_size, image_depth=args.img_depth, train=True, transform=transforms.ToTensor()) test_dataset = LoadDataset(dataset_folder_path=args.data_folder, image_size=args.img_size, image_depth=args.img_depth, train=False, transform=transforms.ToTensor()) train_sampler = torch.utils.data.distributed.DistributedSample