From d67292406e73a4f4fcfd6a7d400bba27dbc26a04 Mon Sep 17 00:00:00 2001 From: Yi Pan Date: Tue, 2 Jul 2024 22:26:20 +0800 Subject: [PATCH] feat(machinelearning): add cuda support for project 5. --- machinelearning/autograder.py | 77 ++++++------- machinelearning/backend.py | 205 ++++++++++------------------------ 2 files changed, 95 insertions(+), 187 deletions(-) diff --git a/machinelearning/autograder.py b/machinelearning/autograder.py index 6f6a6bd..17be080 100644 --- a/machinelearning/autograder.py +++ b/machinelearning/autograder.py @@ -1,5 +1,3 @@ -# A custom autograder for this project - ################################################################################ # A mini-framework for autograding ################################################################################ @@ -222,12 +220,11 @@ def main(): # Tests begin here ################################################################################ -import numpy as np +import torch import matplotlib import contextlib from torch import nn, Tensor -import torch import backend def check_dependencies(): @@ -240,9 +237,9 @@ def check_dependencies(): for t in range(400): angle = t * 0.05 - x = np.sin(angle) - y = np.cos(angle) - line.set_data([x,-x], [y,-y]) + x = torch.sin(torch.tensor(angle)) + y = torch.cos(torch.tensor(angle)) + line.set_data([x.item(), -x.item()], [y.item(), -y.item()]) fig.canvas.draw_idle() fig.canvas.start_event_loop(1e-3) @@ -279,7 +276,7 @@ def verify_node(node, expected_type, expected_shape, method_name): assert False, "If you see this message, please report a bug in the autograder" if expected_type != 'loss': - assert all([(expected is '?' or actual == expected) for (actual, expected) in zip(node.detach().numpy().shape, expected_shape)]), ( + assert all([(expected == '?' or actual == expected) for (actual, expected) in zip(node.shape, expected_shape)]), ( "{} should return an object with shape {}, got {}".format( method_name, expected_shape, node.shape)) @@ -288,7 +285,7 @@ def check_perceptron(tracker): import models print("Sanity checking perceptron...") - np_random = np.random.RandomState(0) + torch.manual_seed(0) # Check that the perceptron weights are initialized to a single vector with `dimensions` entries. for dimensions in range(1, 10): @@ -306,16 +303,16 @@ def check_perceptron(tracker): # Check that run returns a Tensor, and that the score in the node is correct for dimensions in range(1, 10): p = models.PerceptronModel(dimensions) - point = np_random.uniform(-10, 10, (1, dimensions)) - score = p.run(Tensor(point)) + point = torch.empty((1, dimensions)).uniform_(-10, 10) + score = p.run(point) verify_node(score, 'tensor', (1,), "PerceptronModel.run()") calculated_score = score.item() # Compare run output to actual value for param in p.parameters(): - expected_score = float(np.dot(point.flatten(), param.detach().numpy().flatten())) + expected_score = float(torch.dot(point.flatten(), param.detach().flatten())) - assert np.isclose(calculated_score, expected_score), ( + assert torch.isclose(torch.tensor(calculated_score), torch.tensor(expected_score)), ( "The score computed by PerceptronModel.run() ({:.4f}) does not match the expected score ({:.4f})".format( calculated_score, expected_score)) @@ -323,14 +320,14 @@ def check_perceptron(tracker): # case when a point lies exactly on the decision boundary for dimensions in range(1, 10): p = models.PerceptronModel(dimensions) - random_point = np_random.uniform(-10, 10, (1, dimensions)) - for point in (random_point, np.zeros_like(random_point)): - prediction = p.get_prediction(Tensor(point)) + random_point = torch.empty((1, dimensions)).uniform_(-10, 10) + for point in (random_point, torch.zeros_like(random_point)): + prediction = p.get_prediction(point) assert prediction == 1 or prediction == -1, ( "PerceptronModel.get_prediction() should return 1 or -1, not {}".format( prediction)) - expected_prediction = np.where(np.dot(point, p.get_weights().data.T) >= 0, 1, -1).item() + expected_prediction = torch.where(torch.dot(point.flatten(), p.get_weights().data.T.flatten()) >= 0, torch.tensor(1), torch.tensor(-1)).item() assert prediction == expected_prediction, ( "PerceptronModel.get_prediction() returned {}; expected {}".format( prediction, expected_prediction)) @@ -346,27 +343,27 @@ def check_perceptron(tracker): dimensions = 2 for multiplier in (-5, -2, 2, 5): p = models.PerceptronModel(dimensions) - orig_weights = p.get_weights().data.reshape((1, dimensions)).detach().numpy().copy() - if np.abs(orig_weights).sum() == 0.0: + orig_weights = p.get_weights().data.reshape((1, dimensions)).detach().clone() + if torch.abs(orig_weights).sum() == 0.0: # This autograder test doesn't work when weights are exactly zero continue point = multiplier * orig_weights - sanity_dataset = backend.Custom_Dataset( - x=np.tile(point, (500, 1)), - y=np.ones((500, 1)) * -1.0 + sanity_dataset = backend.CustomDataset( + x=point.repeat((500, 1)), + y=torch.ones((500, 1)) * -1.0 ) p.train(sanity_dataset) - new_weights = p.get_weights().data.reshape((1, dimensions)).detach().numpy() + new_weights = p.get_weights().data.reshape((1, dimensions)).detach().clone() if multiplier < 0: expected_weights = orig_weights else: expected_weights = orig_weights - point - if not np.all(new_weights == expected_weights): + if not torch.equal(new_weights, expected_weights): print() print("Initial perceptron weights were: [{:.4f}, {:.4f}]".format( orig_weights[0,0], orig_weights[0,1])) @@ -390,7 +387,7 @@ def check_perceptron(tracker): assert dataset.epoch != 0, "Perceptron code never iterated over the training data" - accuracy = np.mean(np.where(np.dot(dataset.x, model.get_weights().data.T) >= 0.0, 1.0, -1.0) == dataset.y) + accuracy = torch.mean((torch.where(torch.matmul(torch.tensor(dataset.x, dtype=torch.float32), model.get_weights().data.T) >= 0.0, 1.0, -1.0) == torch.tensor(dataset.y)).float()) if accuracy < 1.0: print("The weights learned by your perceptron correctly classified {:.2%} of training examples".format(accuracy)) print("To receive full points for this question, your perceptron must converge to 100% accuracy") @@ -441,7 +438,7 @@ def check_regression(tracker): error = labels - train_predicted sanity_loss = torch.mean((error.detach())**2) - assert np.isclose(train_loss, sanity_loss), ( + assert torch.isclose(torch.tensor(train_loss), sanity_loss), ( "RegressionModel.get_loss() returned a loss of {:.4f}, " "but the autograder computed a loss of {:.4f} " "based on the output of RegressionModel()".format( @@ -484,9 +481,9 @@ def check_digit_classification(tracker): model.train(dataset) - test_logits = model.run(torch.tensor(dataset.test_images)).data - test_predicted = np.argmax(test_logits, axis=1).detach().numpy() - test_accuracy = np.mean(test_predicted == dataset.test_labels) + test_logits = model.run(torch.tensor(dataset.test_images)).detach().cpu() + test_predicted = torch.argmax(test_logits, axis=1) + test_accuracy = torch.mean(torch.eq(test_predicted, torch.tensor(dataset.test_labels)).float()) accuracy_threshold = 0.97 if test_accuracy >= accuracy_threshold: @@ -553,10 +550,10 @@ def check_convolution(tracker): dataset = backend.DigitClassificationDataset2(model) def conv2d(a, f): - s = f.shape + tuple(np.subtract(a.shape, f.shape) + 1) - strd = np.lib.stride_tricks.as_strided - subM = strd(a, shape = s, strides = a.strides * 2) - return np.einsum('ij,ijkl->kl', f, subM) + s = f.shape + tuple(torch.tensor(a.shape) - torch.tensor(f.shape) + 1) + strd = torch.as_strided + subM = strd(a, size = s, stride = a.stride() * 2) + return torch.einsum('ij,ijkl->kl', f, subM) detected_parameters = None @@ -575,20 +572,20 @@ def check_convolution(tracker): assert grad_y[0] != None, "Node returned from RegressionModel.get_loss() does not depend on the provided labels (y)" for matrix_size in (2, 4, 6): #Test 3 random convolutions to test convolve() function - weights = np.random.rand(2,2) - input = np.random.rand(matrix_size, matrix_size) - student_output = models.Convolve(torch.Tensor(input), torch.Tensor(weights)) + weights = torch.rand(2,2) + input = torch.rand(matrix_size, matrix_size) + student_output = models.Convolve(input, weights) actual_output = conv2d(input,weights) - assert np.isclose(student_output, actual_output).all(), "The convolution returned by Convolve() does not match expected output" + assert torch.isclose(student_output, actual_output).all(), "The convolution returned by Convolve() does not match expected output" tracker.add_points(1/2) # Partial credit for testing whether convolution function works model.train(dataset) - test_logits = model.run(torch.tensor(dataset.test_images)).data - test_predicted = np.argmax(test_logits, axis=1).detach().numpy() - test_accuracy = np.mean(test_predicted == dataset.test_labels) + test_logits = model.run(torch.tensor(dataset.test_images)).detach().cpu() + test_predicted = torch.argmax(test_logits, axis=1) + test_accuracy = torch.mean(torch.eq(test_predicted, torch.tensor(dataset.test_labels)).float()) accuracy_threshold = 0.80 if test_accuracy >= accuracy_threshold: diff --git a/machinelearning/backend.py b/machinelearning/backend.py index 29eb714..415f019 100644 --- a/machinelearning/backend.py +++ b/machinelearning/backend.py @@ -1,7 +1,6 @@ import collections import os import time -import os import matplotlib.pyplot as plt import numpy as np @@ -10,7 +9,6 @@ from torch import nn import torch from torch.utils.data import Dataset, DataLoader - use_graphics = True def maybe_sleep_and_close(seconds): @@ -38,37 +36,24 @@ def get_data_path(filename): raise Exception("Could not find data file: {}".format(filename)) return path -class Custom_Dataset(Dataset): +class CustomDataset(Dataset): def __init__(self, x, y, transform=None): - assert isinstance(x, np.ndarray) - assert isinstance(y, np.ndarray) - assert np.issubdtype(x.dtype, np.floating) - assert np.issubdtype(y.dtype, np.floating) - assert x.ndim == 2 - assert y.ndim == 2 - assert x.shape[0] == y.shape[0] - self.x = x - self.y = y + self.x = torch.tensor(x, dtype=torch.float32) + self.y = torch.tensor(y, dtype=torch.float32) self.transform = transform def __len__(self): return len(self.x) def __getitem__(self, idx): - if torch.is_tensor(idx): - idx = idx.tolist() - - label = self.y[idx] x = self.x[idx] - - sample = {'x': torch.Tensor(x), 'label': torch.Tensor(label)} + y = self.y[idx] + sample = {'x': x, 'label': y} if self.transform: sample = self.transform(sample) - - return sample - + return sample def get_validation_accuracy(self): raise NotImplementedError( @@ -76,7 +61,7 @@ class Custom_Dataset(Dataset): "In this assignment, only the Digit Classification and Language " "Identification datasets have validation data.") -class PerceptronDataset(Custom_Dataset): +class PerceptronDataset(CustomDataset): def __init__(self, model): points = 500 x = np.hstack([np.random.randn(points, 2), np.ones((points, 1))]) @@ -103,9 +88,7 @@ class PerceptronDataset(Custom_Dataset): self.line = line self.text = text self.last_update = time.time() - - def __getitem__(self, idx): self.epoch += 1 @@ -115,8 +98,6 @@ class PerceptronDataset(Custom_Dataset): x = self.x[idx] y = self.y[idx] - - if use_graphics and time.time() - self.last_update > 0.01: w = self.model.get_weights().data.flatten() limits = self.limits @@ -133,9 +114,9 @@ class PerceptronDataset(Custom_Dataset): self.fig.canvas.start_event_loop(1e-3) self.last_update = time.time() - return {'x': torch.tensor(x, dtype=torch.float32), 'label': torch.tensor(y, dtype=torch.float32)} - -class RegressionDataset(Custom_Dataset): + return {'x': x, 'label': y} + +class RegressionDataset(CustomDataset): def __init__(self, model): x = np.expand_dims(np.linspace(-2 * np.pi, 2 * np.pi, num=200), axis=1) np.random.RandomState(0).shuffle(x) @@ -161,13 +142,8 @@ class RegressionDataset(Custom_Dataset): self.text = text self.last_update = time.time() - def __len__(self): - return len(self.x) - def __getitem__(self, idx): - data = super().__getitem__(idx) - x = data['x'] y = data['label'] @@ -175,18 +151,17 @@ class RegressionDataset(Custom_Dataset): if use_graphics and time.time() - self.last_update > 0.1: predicted = self.model(torch.tensor(self.x, dtype=torch.float32)).data - loss = self.model.get_loss( - x, y).data + loss = self.model.get_loss(x, y).data self.learned.set_data(self.x[self.argsort_x], predicted[self.argsort_x]) self.text.set_text("processed: {:,}\nloss: {:.6f}".format( self.processed, loss)) self.fig.canvas.draw_idle() self.fig.canvas.start_event_loop(1e-3) self.last_update = time.time() - + return {'x': x, 'label': y} -class DigitClassificationDataset(Custom_Dataset): +class DigitClassificationDataset(CustomDataset): def __init__(self, model): mnist_path = get_data_path("mnist.npz") @@ -252,34 +227,25 @@ class DigitClassificationDataset(Custom_Dataset): self.status = status self.last_update = time.time() - def __getitem__(self, idx): - - data = super().__getitem__(idx) - x = data['x'] y = data['label'] if use_graphics and time.time() - self.last_update > 1: - dev_logits = self.model.run(torch.tensor(self.dev_images)).data - dev_predicted = np.argmax(dev_logits, axis=1).detach().numpy() - dev_probs = np.exp(nn.functional.log_softmax(dev_logits)) + dev_logits = self.model.run(torch.tensor(self.dev_images, dtype=torch.float32)).detach().cpu() + dev_predicted = torch.argmax(dev_logits, axis=1) + dev_probs = torch.exp(nn.functional.log_softmax(dev_logits, dim=1)) - dev_accuracy = np.mean(dev_predicted == self.dev_labels) + dev_accuracy = torch.mean(torch.eq(dev_predicted, torch.tensor(self.dev_labels)).float()) self.status.set_text( - "validation accuracy: " - "{:.2%}".format( - dev_accuracy)) + "validation accuracy: {:.2%}".format(dev_accuracy)) for i in range(10): predicted = dev_predicted[self.dev_labels == i] probs = dev_probs[self.dev_labels == i][:, i] - linspace = np.linspace( - 0, len(probs) - 1, self.samples).astype(int) + linspace = np.linspace(0, len(probs) - 1, self.samples).astype(int) indices = probs.argsort()[linspace] - for j, (prob, image) in enumerate(zip( - probs[indices], - self.dev_images[self.dev_labels == i][indices])): + for j, (prob, image) in enumerate(zip(probs[indices], self.dev_images[self.dev_labels == i][indices])): self.images[i][j].set_data(image.reshape((28, 28))) left = prob * (self.width - 1) * 28 if predicted[indices[j]] == i: @@ -287,29 +253,22 @@ class DigitClassificationDataset(Custom_Dataset): self.texts[i][j].set_text("") else: self.images[i][j].set_cmap("Reds") - self.texts[i][j].set_text(predicted[indices[j]]) + self.texts[i][j].set_text(predicted[indices[j]].detach().cpu().numpy()) self.texts[i][j].set_x(left + 14) self.images[i][j].set_extent([left, left + 28, 0, 28]) self.fig.canvas.draw_idle() self.fig.canvas.start_event_loop(1e-3) self.last_update = time.time() - - if(self.num_items == len(self.x)): - self.current_accuracy = self.num_right_items/len(self.x) - self.num_right_items = 0 - self.epoch += 1 return {'x': x, 'label': y} def get_validation_accuracy(self): - dev_logits = self.model.run(torch.tensor(self.dev_images)).data - dev_predicted = np.argmax(dev_logits, axis=1).detach().numpy() - dev_probs = np.exp(nn.functional.log_softmax(dev_logits)) - - dev_accuracy = np.mean(dev_predicted == self.dev_labels) + dev_logits = self.model.run(torch.tensor(self.dev_images, dtype=torch.float32)).data + dev_predicted = torch.argmax(dev_logits, axis=1).detach() + dev_accuracy = (dev_predicted == self.dev_labels).mean() return dev_accuracy -class LanguageIDDataset(Custom_Dataset): +class LanguageIDDataset(CustomDataset): def __init__(self, model): self.model = model @@ -330,7 +289,7 @@ class LanguageIDDataset(Custom_Dataset): self.test_buckets = data['test_buckets'] self.epoch = 0 - self.bucket_weights = self.train_buckets[:,1] - self.train_buckets[:,0] + self.bucket_weights = self.train_buckets[:, 1] - self.train_buckets[:, 0] self.bucket_weights = self.bucket_weights / float(self.bucket_weights.sum()) self.chars_print = self.chars @@ -358,14 +317,12 @@ alphabet above have been substituted with ASCII symbols.""".strip()) max_word_len = self.dev_x.shape[1] max_lang_len = max([len(x) for x in self.language_names]) - self.predicted_template = u"Pred: {: 1: - dev_logits = self.model.run(torch.tensor(self.dev_images)).data - dev_predicted = np.argmax(dev_logits, axis=1).detach().numpy() - dev_probs = np.exp(nn.functional.log_softmax(dev_logits)) + dev_logits = self.model.run(torch.tensor(self.dev_images, dtype=torch.float32)).detach().cpu() + dev_predicted = torch.argmax(dev_logits, axis=1) + dev_probs = torch.exp(nn.functional.log_softmax(dev_logits, dim=1)) - dev_accuracy = np.mean(dev_predicted == self.dev_labels) - self.status.set_text( - "validation accuracy: " - "{:.2%}".format( - dev_accuracy)) + dev_accuracy = torch.mean(torch.eq(dev_predicted, torch.tensor(self.dev_labels)).float()) + self.status.set_text("validation accuracy: {:.2%}".format(dev_accuracy)) for i in range(10): predicted = dev_predicted[self.dev_labels == i] probs = dev_probs[self.dev_labels == i][:, i] - linspace = np.linspace( - 0, len(probs) - 1, self.samples).astype(int) + linspace = np.linspace(0, len(probs) - 1, self.samples).astype(int) indices = probs.argsort()[linspace] - for j, (prob, image) in enumerate(zip( - probs[indices], - self.dev_images[self.dev_labels == i][indices])): + for j, (prob, image) in enumerate(zip(probs[indices], self.dev_images[self.dev_labels == i][indices])): self.images[i][j].set_data(image.reshape((28, 28))) - left = prob * (self.width - 1) * 28 + left = (prob * (self.width - 1) * 28).detach().cpu().numpy() if predicted[indices[j]] == i: self.images[i][j].set_cmap("Greens") self.texts[i][j].set_text("") else: self.images[i][j].set_cmap("Reds") - self.texts[i][j].set_text(predicted[indices[j]]) + self.texts[i][j].set_text(predicted[indices[j]].detach().cpu().numpy()) self.texts[i][j].set_x(left + 14) self.images[i][j].set_extent([left, left + 28, 0, 28]) self.fig.canvas.draw_idle() self.fig.canvas.start_event_loop(1e-3) self.last_update = time.time() - - if(self.num_items == len(self.x)): - self.current_accuracy = self.num_right_items/len(self.x) - self.num_right_items = 0 - self.epoch += 1 return {'x': x, 'label': y} def get_validation_accuracy(self): - dev_logits = self.model.run(torch.tensor(self.dev_images)).data - dev_predicted = np.argmax(dev_logits, axis=1).detach().numpy() - dev_probs = np.exp(nn.functional.log_softmax(dev_logits)) - - dev_accuracy = np.mean(dev_predicted == self.dev_labels) + dev_logits = self.model.run(torch.tensor(self.dev_images, dtype=torch.float32)).data + dev_predicted = torch.argmax(dev_logits, axis=1).detach() + dev_accuracy = torch.mean(torch.eq(dev_predicted, torch.tensor(self.dev_labels)).float()) return dev_accuracy - - def main(): import models model = models.PerceptronModel(3) @@ -598,4 +510,3 @@ def main(): if __name__ == "__main__": main() -