503 lines
14 KiB
Plaintext
503 lines
14 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "a5a8f05e",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Question 4 (10 marks)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "8ddab76f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import torch\n",
|
|
"import torch.nn.functional as F\n",
|
|
"from ca_utils import ResNet, BasicBlock"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "b52a47c6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"model = ResNet(block=BasicBlock, layers=[1, 1, 1], num_classes=10)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "15a25e00",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Load the Model"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"id": "af4de07b",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"ResNet(\n",
|
|
" (conv1): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
|
|
" (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
|
" (relu): ReLU(inplace=True)\n",
|
|
" (layer1): Sequential(\n",
|
|
" (0): BasicBlock(\n",
|
|
" (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
|
|
" (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
|
" (relu): ReLU(inplace=True)\n",
|
|
" (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
|
|
" (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (layer2): Sequential(\n",
|
|
" (0): BasicBlock(\n",
|
|
" (conv1): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
|
|
" (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
|
" (relu): ReLU(inplace=True)\n",
|
|
" (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
|
|
" (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
|
" (downsample): Sequential(\n",
|
|
" (0): Conv2d(16, 32, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
|
|
" (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (layer3): Sequential(\n",
|
|
" (0): BasicBlock(\n",
|
|
" (conv1): Conv2d(32, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
|
|
" (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
|
" (relu): ReLU(inplace=True)\n",
|
|
" (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
|
|
" (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
|
" (downsample): Sequential(\n",
|
|
" (0): Conv2d(32, 64, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
|
|
" (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
|
|
" (fc): Linear(in_features=64, out_features=10, bias=True)\n",
|
|
")"
|
|
]
|
|
},
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"checkpoint = torch.load(\"data/weights_resnet.pth\", map_location=torch.device('cpu'))\n",
|
|
"\n",
|
|
"model.load_state_dict(checkpoint)\n",
|
|
"model.eval()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "98c03b18",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Question 5 (15 marks)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "3024a7a8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import torchvision\n",
|
|
"from torch.utils.data import DataLoader\n",
|
|
"from torchvision import transforms\n",
|
|
"\n",
|
|
"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"id": "357891cd",
|
|
"metadata": {
|
|
"scrolled": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"image_transform = transforms.Compose(\n",
|
|
" [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
|
|
"\n",
|
|
"# image_transform = transforms.Compose([\n",
|
|
"# # transforms.Resize(256),\n",
|
|
"# # transforms.CenterCrop(224),\n",
|
|
"# transforms.ToTensor(),\n",
|
|
"# transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
|
|
"# ])\n",
|
|
"\n",
|
|
"test_data = torchvision.datasets.ImageFolder('data/EXCV10/val/', transform=image_transform)\n",
|
|
"test_loader = DataLoader(test_data, batch_size=64, shuffle=False, num_workers=4, pin_memory=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "0eb07988",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Method 1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"id": "b9d88037",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"id": "6a30d88c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def m1_test_cnn(model, test_loader):\n",
|
|
"\n",
|
|
" model.to(device)\n",
|
|
" model.eval()\n",
|
|
"\n",
|
|
" correct = 0\n",
|
|
" total = 0\n",
|
|
" all_predicted_labels = []\n",
|
|
"\n",
|
|
" with torch.no_grad():\n",
|
|
" for images, labels in test_loader:\n",
|
|
"\n",
|
|
" # Make predictions\n",
|
|
" images, labels = images.to(device), labels.to(device)\n",
|
|
" outputs = model(images)\n",
|
|
"\n",
|
|
" _, predicted = torch.max(outputs.data, 1)\n",
|
|
"\n",
|
|
" # Save results\n",
|
|
" total += labels.size(0)\n",
|
|
" correct += (predicted == labels).sum().item()\n",
|
|
" \n",
|
|
" all_predicted_labels.append(predicted.cpu().numpy())\n",
|
|
"\n",
|
|
" accuracy = 100 * correct / total\n",
|
|
" all_predicted_labels = np.concatenate(all_predicted_labels)\n",
|
|
"\n",
|
|
" return all_predicted_labels, accuracy"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"id": "3fcb0a3a",
|
|
"metadata": {
|
|
"scrolled": false
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Test Accuracy: 70.05%\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"m1_predicted_labels, m1_test_accuracy = m1_test_cnn(model, test_loader)\n",
|
|
"print(f'Test Accuracy: {m1_test_accuracy}%')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "75272741",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Put Students' implementations here"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"id": "47bd1202",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def test_cnn(model, test_loader, device='cpu'):\n",
|
|
" model.to(device)\n",
|
|
" model.eval() \n",
|
|
" total = 0\n",
|
|
" correct_num = 0\n",
|
|
" all_predicted_labels = []\n",
|
|
"\n",
|
|
" with torch.no_grad(): # No need to track gradients for testing\n",
|
|
" for images, labels in test_loader:\n",
|
|
" images, labels = images.to(device), labels.to(device)\n",
|
|
" outputs = model(images)\n",
|
|
" _, predicted = torch.max(outputs.data, 1)\n",
|
|
" total += labels.size(0)\n",
|
|
" correct_num += (predicted == labels).sum().item() \n",
|
|
" all_predicted_labels.append(predicted.cpu().numpy())\n",
|
|
"\n",
|
|
" accuracy = (correct_num / total) * 100\n",
|
|
" all_predicted_labels = np.concatenate(all_predicted_labels)\n",
|
|
" return all_predicted_labels, accuracy"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"id": "1b0db3bd",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Test Accuracy: 70.05%\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"predicted_labels, test_accuracy = test_cnn(model, test_loader)\n",
|
|
"print(f'Test Accuracy: {test_accuracy}%')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "985e4f91",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Test (Should output ALL PASS)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"id": "694097e2",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Test accuracy: 70.05\n",
|
|
"Score 90%: 13.5\n",
|
|
"ALL PASS\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"assert np.allclose(predicted_labels, m1_predicted_labels)\n",
|
|
"assert np.allclose(test_accuracy, m1_test_accuracy)\n",
|
|
"\n",
|
|
"print(\"Test accuracy: \", test_accuracy)\n",
|
|
"\n",
|
|
"if (test_accuracy >= 75):\n",
|
|
" print(\"Score 100%:\", 15 * 1.0)\n",
|
|
"elif (test_accuracy >= 70):\n",
|
|
" print(\"Score 90%:\", 15 * 0.90)\n",
|
|
"elif (test_accuracy >= 65):\n",
|
|
" print(\"Score 80%:\", 15 * 0.80)\n",
|
|
"elif (test_accuracy >= 60):\n",
|
|
" print(\"Score 70%:\", 15 * 0.70)\n",
|
|
"elif (test_accuracy >= 55):\n",
|
|
" print(\"Score 60%:\", 15 * 0.60)\n",
|
|
"elif (test_accuracy >= 50):\n",
|
|
" print(\"Score 50%:\", 15 * 0.50)\n",
|
|
"else:\n",
|
|
" print(\"Accuracy less than 50%\")\n",
|
|
"print(\"ALL PASS\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cef7dc17",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Question 6 (6 marks)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"id": "0990f3b2",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"true_labels = []\n",
|
|
"\n",
|
|
"for images, labels in test_loader:\n",
|
|
" images, labels = images.to(device), labels.to(device)\n",
|
|
" true_labels.extend(labels.cpu().numpy())\n",
|
|
" \n",
|
|
"true_labels = np.array(true_labels)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"id": "8da35032",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def m1_compute_confusion_matrix(true, predictions):\n",
|
|
" unique_labels = np.unique(np.concatenate((true, predictions)))\n",
|
|
"\n",
|
|
" confusion_mat = np.zeros((len(unique_labels), len(unique_labels)), dtype=np.int64)\n",
|
|
"\n",
|
|
" label_to_index = {label: index for index,\n",
|
|
" label in enumerate(unique_labels)}\n",
|
|
"\n",
|
|
" for t, p in zip(true, predictions):\n",
|
|
" t_index = label_to_index[t]\n",
|
|
" p_index = label_to_index[p]\n",
|
|
" confusion_mat[t_index][p_index] += 1\n",
|
|
"\n",
|
|
" return confusion_mat"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 14,
|
|
"id": "16b6f9e7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"m1_confusion_matrix = m1_compute_confusion_matrix(true_labels, m1_predicted_labels)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 17,
|
|
"id": "60591999",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from sklearn.metrics import confusion_matrix\n",
|
|
"\n",
|
|
"def m2_compute_confusion_matrix(true, predictions):\n",
|
|
" return confusion_matrix(true, predictions)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 18,
|
|
"id": "44b131f0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"m2_confusion_matrix = m2_compute_confusion_matrix(true_labels, m1_predicted_labels)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "dd78bea6",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Put Students' implementations here"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 19,
|
|
"id": "1dce952c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def compute_confusion_matrix(true, predictions):\n",
|
|
" unique_labels = np.unique(np.concatenate((true, predictions)))\n",
|
|
" confusion_matrix = np.zeros((len(unique_labels), len(unique_labels)), dtype=np.int64)\n",
|
|
" for i, true_label in enumerate(unique_labels):\n",
|
|
" for j, predicted_label in enumerate(unique_labels):\n",
|
|
" confusion_matrix[i, j] = np.sum((true == true_label) & (predictions == predicted_label))\n",
|
|
" return confusion_matrix"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 20,
|
|
"id": "1945d637",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"confusion_matrix = compute_confusion_matrix(true_labels, predicted_labels)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "3e1fd6eb",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Test (Should output ALL PASS)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 21,
|
|
"id": "6c87f7d6",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"ALL PASS\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"assert np.allclose(m1_confusion_matrix, m2_confusion_matrix)\n",
|
|
"assert np.allclose(confusion_matrix, m1_confusion_matrix)\n",
|
|
"\n",
|
|
"print(\"ALL PASS\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d9bb6316",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "what",
|
|
"language": "python",
|
|
"name": "what"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.8.16"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|