# DualOpt
Dual Optimizer Training
A variant of the [SWATS training paradigm](https://arxiv.org/abs/1712.07628) which uses two optimizers for training.
## Install
```bash
$ pip install dualopt
```
## Usage
### Image Classification
```python
import dualopt, torch
from dualopt import classification
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
num_classes = 10
#define model
#define datasets and dataloaders
top1 = [] #top1 accuracy
top5 = [] #top5 accuracy
traintime = []
testtime = []
counter = 20 # number of epochs without any improvement in accuracy before we stop training for each optimizer
PATH = 'saved_model.pth' #path to save model
classification(model, trainloader, testloader, device, PATH, top1, top5, traintime, testtime, num_classes = num_classes, set_counter = counter)
print('Finished Training')
print("Results")
print(f"Top 1 Accuracy: {max(top1):.2f} -Top 5 Accuracy : {max(top5):.2f} - Train Time: {min(traintime):.0f} -Test Time: {min(testtime):.0f}\n")
```
# Post-Training
Experiments show that we get good results when training using data augmentations such as Trivial Augment. We found that subsequent post-training without using any data augmentations can further improve the results.
## Usage
```python
import dualopt, torch, torchvision
import torchvision.transforms as transforms
from dualopt import classification
from dualopt import post_train
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
num_classes = 10
#define model
#set batch size according to GPU
batch_size = 512
# transforms
transform_train_1 = transforms.Compose(
[ transforms.RandomHorizontalFlip(p=0.5),
transforms.TrivialAugmentWide(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))])
transform_train_2 = transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))])
transform_test = transforms.Compose(
[ transforms.ToTensor(),
transforms.Normalize((0.4941, 0.4853, 0.4507), (0.2468, 0.2430, 0.2618))])
#Dataset
trainset_1 = torchvision.datasets.CIFAR10(root='/workspace/', train=True, download=True, transform=transform_train_1)
trainloader_1 = torch.utils.data.DataLoader(trainset_1, batch_size=batch_size, shuffle=True, num_workers=2, pin_memory=True, prefetch_factor=2, persistent_workers=2) #trainloader with augmentations
trainset_2 = torchvision.datasets.CIFAR10(root='/workspace/', train=True, download=True, transform=transform_train_2)
trainloader_2 = torch.utils.data.DataLoader(trainset_2, batch_size=batch_size, shuffle=True, num_workers=2, pin_memory=True, prefetch_factor=2, persistent_workers=2) #trainloader for post-training without augmentations
testset = torchvision.datasets.CIFAR10(root='/workspace/', train=False, download=True, transform=transform_test)
testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2, pin_memory=True, prefetch_factor=2, persistent_workers=2)
top1 = [] #top1 accuracy
top5 = [] #top5 accuracy
traintime = []
testtime = []
counter = 20 # number of epochs without any improvement in accuracy before we stop training for each optimizer
PATH = 'saved_model.pth' #path to save model
classification(model, trainloader_1, testloader, device, PATH, top1, top5, traintime, testtime, num_classes = num_classes, set_counter = counter)
print('Finished Training')
model.load_state_dict(torch.load(PATH))
post_train(model, trainloader_2, testloader, device, PATH, top1, top5, traintime, testtime, num_classes = num_classes, set_counter = counter)
print('Finished Training')
print("Results")
print(f"Top 1 Accuracy: {max(top1):.2f} -Top 5 Accuracy : {max(top5):.2f} - Train Time: {min(traintime):.0f} -Test Time: {min(testtime):.0f}\n")
```
#### Cite the following paper
```
@misc{jeevan2022convolutional,
title={Convolutional Xformers for Vision},
author={Pranav Jeevan and Amit sethi},
year={2022},
eprint={2201.10271},
archivePrefix={arXiv},
primaryClass={cs.CV}
}
```
Raw data
{
"_id": null,
"home_page": "https://github.com/pranavphoenix/DualOpt",
"name": "dualopt",
"maintainer": "",
"docs_url": null,
"requires_python": "",
"maintainer_email": "",
"keywords": "artificial intelligence,machine learning,optimizer,training,image classificationsemantic segmentation,image super-resolution",
"author": "Pranav Jeevan",
"author_email": "pranav13phoenix@gmail.com",
"download_url": "https://files.pythonhosted.org/packages/ba/dc/d53d71e55ebed8da34ba26f6baffd759db53964bcad73d002a1b1fe1d166/dualopt-0.1.8.tar.gz",
"platform": null,
"description": "# DualOpt\r\nDual Optimizer Training \r\n\r\nA variant of the [SWATS training paradigm](https://arxiv.org/abs/1712.07628) which uses two optimizers for training. \r\n\r\n## Install\r\n\r\n```bash\r\n$ pip install dualopt\r\n```\r\n\r\n## Usage\r\n### Image Classification\r\n\r\n```python\r\nimport dualopt, torch\r\nfrom dualopt import classification\r\n\r\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\r\n\r\nnum_classes = 10\r\n\r\n#define model\r\n#define datasets and dataloaders\r\n\r\ntop1 = [] #top1 accuracy\r\ntop5 = [] #top5 accuracy\r\ntraintime = []\r\ntesttime = []\r\ncounter = 20 # number of epochs without any improvement in accuracy before we stop training for each optimizer\r\n\r\nPATH = 'saved_model.pth' #path to save model \r\n\r\nclassification(model, trainloader, testloader, device, PATH, top1, top5, traintime, testtime, num_classes = num_classes, set_counter = counter)\r\n\r\nprint('Finished Training')\r\nprint(\"Results\")\r\nprint(f\"Top 1 Accuracy: {max(top1):.2f} -Top 5 Accuracy : {max(top5):.2f} - Train Time: {min(traintime):.0f} -Test Time: {min(testtime):.0f}\\n\")\r\n```\r\n# Post-Training\r\n\r\nExperiments show that we get good results when training using data augmentations such as Trivial Augment. We found that subsequent post-training without using any data augmentations can further improve the results. \r\n\r\n## Usage\r\n\r\n\r\n```python\r\nimport dualopt, torch, torchvision\r\nimport torchvision.transforms as transforms\r\nfrom dualopt import classification\r\nfrom dualopt import post_train\r\n\r\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\r\n\r\nnum_classes = 10\r\n\r\n#define model\r\n\r\n#set batch size according to GPU \r\nbatch_size = 512\r\n\r\n# transforms\r\ntransform_train_1 = transforms.Compose(\r\n [ transforms.RandomHorizontalFlip(p=0.5),\r\n transforms.TrivialAugmentWide(),\r\n transforms.ToTensor(),\r\n transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))])\r\n\r\ntransform_train_2 = transforms.Compose(\r\n [ \r\n transforms.ToTensor(),\r\n transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))])\r\n\r\n\r\ntransform_test = transforms.Compose(\r\n [ transforms.ToTensor(),\r\n transforms.Normalize((0.4941, 0.4853, 0.4507), (0.2468, 0.2430, 0.2618))])\r\n\r\n#Dataset\r\ntrainset_1 = torchvision.datasets.CIFAR10(root='/workspace/', train=True, download=True, transform=transform_train_1)\r\ntrainloader_1 = torch.utils.data.DataLoader(trainset_1, batch_size=batch_size, shuffle=True, num_workers=2, pin_memory=True, prefetch_factor=2, persistent_workers=2) #trainloader with augmentations\r\n\r\ntrainset_2 = torchvision.datasets.CIFAR10(root='/workspace/', train=True, download=True, transform=transform_train_2)\r\ntrainloader_2 = torch.utils.data.DataLoader(trainset_2, batch_size=batch_size, shuffle=True, num_workers=2, pin_memory=True, prefetch_factor=2, persistent_workers=2) #trainloader for post-training without augmentations\r\n\r\ntestset = torchvision.datasets.CIFAR10(root='/workspace/', train=False, download=True, transform=transform_test)\r\ntestloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2, pin_memory=True, prefetch_factor=2, persistent_workers=2)\r\n\r\n\r\ntop1 = [] #top1 accuracy\r\ntop5 = [] #top5 accuracy\r\ntraintime = []\r\ntesttime = []\r\ncounter = 20 # number of epochs without any improvement in accuracy before we stop training for each optimizer\r\n\r\nPATH = 'saved_model.pth' #path to save model \r\n\r\nclassification(model, trainloader_1, testloader, device, PATH, top1, top5, traintime, testtime, num_classes = num_classes, set_counter = counter)\r\nprint('Finished Training')\r\n\r\nmodel.load_state_dict(torch.load(PATH))\r\n\r\npost_train(model, trainloader_2, testloader, device, PATH, top1, top5, traintime, testtime, num_classes = num_classes, set_counter = counter)\r\nprint('Finished Training')\r\n\r\nprint(\"Results\")\r\nprint(f\"Top 1 Accuracy: {max(top1):.2f} -Top 5 Accuracy : {max(top5):.2f} - Train Time: {min(traintime):.0f} -Test Time: {min(testtime):.0f}\\n\")\r\n```\r\n\r\n#### Cite the following paper \r\n```\r\n@misc{jeevan2022convolutional,\r\n title={Convolutional Xformers for Vision}, \r\n author={Pranav Jeevan and Amit sethi},\r\n year={2022},\r\n eprint={2201.10271},\r\n archivePrefix={arXiv},\r\n primaryClass={cs.CV}\r\n}\r\n\r\n```\r\n",
"bugtrack_url": null,
"license": "MIT",
"summary": "Dual Optimizer Training",
"version": "0.1.8",
"project_urls": {
"Homepage": "https://github.com/pranavphoenix/DualOpt"
},
"split_keywords": [
"artificial intelligence",
"machine learning",
"optimizer",
"training",
"image classificationsemantic segmentation",
"image super-resolution"
],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "5b6821bf2fc046389428e410d247c7f714fd96edc98ea7e946d44630b24232e2",
"md5": "7b65cab976e7827c5481cfca555663c2",
"sha256": "9b40d14c28a531153b1bcd981a2b0b47268327e4fd4e24b11929d3439bfe0c01"
},
"downloads": -1,
"filename": "dualopt-0.1.8-py3-none-any.whl",
"has_sig": false,
"md5_digest": "7b65cab976e7827c5481cfca555663c2",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": null,
"size": 5011,
"upload_time": "2023-05-18T16:53:07",
"upload_time_iso_8601": "2023-05-18T16:53:07.866063Z",
"url": "https://files.pythonhosted.org/packages/5b/68/21bf2fc046389428e410d247c7f714fd96edc98ea7e946d44630b24232e2/dualopt-0.1.8-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": "",
"digests": {
"blake2b_256": "badcd53d71e55ebed8da34ba26f6baffd759db53964bcad73d002a1b1fe1d166",
"md5": "bfdfb619fdcc9cecc1199f5dd63968d6",
"sha256": "e0d2b3238b8ec56eeb19c809697aefd6d3cc8f3e5c7f4f7de77056cbc82f7ca4"
},
"downloads": -1,
"filename": "dualopt-0.1.8.tar.gz",
"has_sig": false,
"md5_digest": "bfdfb619fdcc9cecc1199f5dd63968d6",
"packagetype": "sdist",
"python_version": "source",
"requires_python": null,
"size": 4675,
"upload_time": "2023-05-18T16:53:09",
"upload_time_iso_8601": "2023-05-18T16:53:09.406847Z",
"url": "https://files.pythonhosted.org/packages/ba/dc/d53d71e55ebed8da34ba26f6baffd759db53964bcad73d002a1b1fe1d166/dualopt-0.1.8.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2023-05-18 16:53:09",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "pranavphoenix",
"github_project": "DualOpt",
"travis_ci": false,
"coveralls": false,
"github_actions": false,
"lcname": "dualopt"
}
JSON
