Torch Basics

Tensors

R Packages

library(tidyverse)
library(torch)
library(mvtnorm)

Tensors

  • Tensors

  • Vector Operations

  • Matrix Operations

  • Linear Regression

What are Tensors?

Mathematics

A tensor describes a relationship between algebraic objects.

Torch

An n-dimensional array with optimized operations.

Creating Tensors

t1 <- torch_tensor(matrix(1:9, ncol = 3))
t2 <- torch_tensor(1:3)
t3 <- torch_tensor(array(1:24, dim = c(4, 3, 2)))

Tensor Operations

Dimensions

t1$shape
#> [1] 3 3
t2$shape
#> [1] 3
t3$shape
#> [1] 4 3 2

Type

t1$dtype
#> torch_Long
t2$dtype
#> torch_Long
t3$dtype
#> torch_Long

Changing Type

t1$to(dtype = torch_float())
#> torch_tensor
#>  1  4  7
#>  2  5  8
#>  3  6  9
#> [ CPUFloatType{3,3} ]
t2$to(dtype = torch_float())
#> torch_tensor
#>  1
#>  2
#>  3
#> [ CPUFloatType{3} ]
t3$to(dtype = torch_float())
#> torch_tensor
#> (1,.,.) = 
#>    1  13
#>    5  17
#>    9  21
#> 
#> (2,.,.) = 
#>    2  14
#>    6  18
#>   10  22
#> 
#> (3,.,.) = 
#>    3  15
#>    7  19
#>   11  23
#> 
#> (4,.,.) = 
#>    4  16
#>    8  20
#>   12  24
#> [ CPUFloatType{4,3,2} ]

Set Type

t1 <- torch_tensor(matrix(1:9, ncol = 3), dtype = torch_float())
t2 <- torch_tensor(1:3, dtype = torch_float())
t3 <- torch_tensor(array(1:24, dim = c(4, 3, 2)), dtype = torch_float())

Device

t1$device
#> torch_device(type='cpu')
t2$device
#> torch_device(type='cpu')
t3$device
#> torch_device(type='cpu')

Set Device

t1 <- torch_tensor(matrix(1:9, ncol = 3), device = "cuda")
t2 <- torch_tensor(1:3, device = "cuda")
t3 <- torch_tensor(array(1:24, dim = c(4, 3, 2)), device = "cuda")

Note

Use if cuda is installed (Nvidia users)

Vector Operations

  • Tensors

  • Vector Operations

  • Matrix Operations

  • Linear Regression

Vectors

t1 <- torch_tensor(1:6)
t2 <- torch_tensor(7:12)
t3 <- torch_tensor(1:3)
t1; t2; t3
#> torch_tensor
#>  1
#>  2
#>  3
#>  4
#>  5
#>  6
#> [ CPULongType{6} ]
#> torch_tensor
#>   7
#>   8
#>   9
#>  10
#>  11
#>  12
#> [ CPULongType{6} ]
#> torch_tensor
#>  1
#>  2
#>  3
#> [ CPULongType{3} ]

Addition/Subtraction

t1$add(t2)
#> torch_tensor
#>   8
#>  10
#>  12
#>  14
#>  16
#>  18
#> [ CPULongType{6} ]

Scalar Multiplication

10*t1
#> torch_tensor
#>  10
#>  20
#>  30
#>  40
#>  50
#>  60
#> [ CPUFloatType{6} ]

Elementwise Multiplication

t1$multiply(t2)
#> torch_tensor
#>   7
#>  16
#>  27
#>  40
#>  55
#>  72
#> [ CPULongType{6} ]

Dot Product

t1$dot(t2)
#> torch_tensor
#> 217
#> [ CPULongType{} ]

Matrix Operations

  • Tensors

  • Vector Operations

  • Matrix Operations

  • Linear Regression

Matrix

t1 <- torch_tensor(matrix(1:9, ncol = 3), dtype = torch_float())
t2 <- torch_tensor(matrix(rpois(9, 3), ncol = 3), dtype = torch_float())
t3 <- torch_tensor(matrix(1:3, nrow = 3), dtype = torch_float())

t1; t2; t3
#> torch_tensor
#>  1  4  7
#>  2  5  8
#>  3  6  9
#> [ CPUFloatType{3,3} ]
#> torch_tensor
#>  0  2  4
#>  2  5  0
#>  2  4  5
#> [ CPUFloatType{3,3} ]
#> torch_tensor
#>  1
#>  2
#>  3
#> [ CPUFloatType{3,1} ]

Transpose

t1$t()
#> torch_tensor
#>  1  2  3
#>  4  5  6
#>  7  8  9
#> [ CPUFloatType{3,3} ]
t2$t()
#> torch_tensor
#>  0  2  2
#>  2  5  4
#>  4  0  5
#> [ CPUFloatType{3,3} ]
t3$t()
#> torch_tensor
#>  1  2  3
#> [ CPUFloatType{1,3} ]

Addition/Subtraction

t1$add(t2)
#> torch_tensor
#>   1   6  11
#>   4  10   8
#>   5  10  14
#> [ CPUFloatType{3,3} ]
t1$subtract(t2)
#> torch_tensor
#>  1  2  3
#>  0  0  8
#>  1  2  4
#> [ CPUFloatType{3,3} ]

Scalar Multiplication

10*t1
#> torch_tensor
#>  10  40  70
#>  20  50  80
#>  30  60  90
#> [ CPUFloatType{3,3} ]

Element-wise Multiplications

t1; t2
#> torch_tensor
#>  1  4  7
#>  2  5  8
#>  3  6  9
#> [ CPUFloatType{3,3} ]
#> torch_tensor
#>  0  2  4
#>  2  5  0
#>  2  4  5
#> [ CPUFloatType{3,3} ]
t1$multiply(t2)
#> torch_tensor
#>   0   8  28
#>   4  25   0
#>   6  24  45
#> [ CPUFloatType{3,3} ]

Matrix Multiplication

t1$matmul(t2)
#> torch_tensor
#>  22  50  39
#>  26  61  48
#>  30  72  57
#> [ CPUFloatType{3,3} ]
t2$matmul(t1)
#> torch_tensor
#>  16  34  52
#>  12  33  54
#>  25  58  91
#> [ CPUFloatType{3,3} ]
t3$t()$matmul(t1)
#> torch_tensor
#>  14  32  50
#> [ CPUFloatType{1,3} ]

Determinant

t1$det()
#> torch_tensor
#> 0
#> [ CPUFloatType{} ]

Inverse

t2$inverse()
#> torch_tensor
#> -0.8929 -0.2143  0.7143
#>  0.3571  0.2857 -0.2857
#>  0.0714 -0.1429  0.1429
#> [ CPUFloatType{3,3} ]

Diagonal Elements

t1$diag()
#> torch_tensor
#>  1
#>  5
#>  9
#> [ CPUFloatType{3} ]
torch_diag(1:3)
#> torch_tensor
#>  1  0  0
#>  0  2  0
#>  0  0  3
#> [ CPULongType{3,3} ]

Matrix of 0/1

torch_zeros(3,3)
#> torch_tensor
#>  0  0  0
#>  0  0  0
#>  0  0  0
#> [ CPUFloatType{3,3} ]
torch_ones(2,3)
#> torch_tensor
#>  1  1  1
#>  1  1  1
#> [ CPUFloatType{2,3} ]

Identity Matrix

torch_eye(n = 4)
#> torch_tensor
#>  1  0  0  0
#>  0  1  0  0
#>  0  0  1  0
#>  0  0  0  1
#> [ CPUFloatType{4,4} ]

Linear Regression

  • Tensors

  • Vector Operations

  • Matrix Operations

  • Linear Regression

Linear Regression

\[ Y_i = 3.85 + 12.3 X_1 - 9.7 X_2 + \varepsilon_i \]

\[ \varepsilon \sim N(0, 2.4) \]

beta <- c(3.85, 12.3, -9.7)
x <- rmvnorm(500, c(3, 8))
xx <- cbind(1, x)
y <- xx%*%beta + rnorm(500, sd = sqrt(2.4))

OLS Estimator

\[ \hat\bbeta = (\bX^\mrT\bX)\inv\bX^\mrT\bY \]

\[ \bX =\left( \begin{array}{ccc} 1 & X_{1,1} & X_{2,1} \\ 1 & X_{1,2} & X_{2,2} \\ \vdots & \vdots & \vdots \\ 1 & X_{1,500} & X_{2,500} \\ \end{array} \right) \]

\[ \bY =\left( \begin{array}{ccc} Y_1 \\ Y_2 \\ \vdots \\ Y_{500} \end{array} \right) \]

Using R

\[ \hat\bbeta = (\bX^\mrT\bX)\inv\bX^\mrT\bY \]

lm(y ~ x)
#> 
#> Call:
#> lm(formula = y ~ x)
#> 
#> Coefficients:
#> (Intercept)           x1           x2  
#>       3.272       12.207       -9.594
solve(t(xx)%*%xx)%*%t(xx)%*%y
#>           [,1]
#> [1,]  3.271945
#> [2,] 12.206737
#> [3,] -9.594003

Using torch

\[ \hat\bbeta = (\bX^\mrT\bX)\inv\bX^\mrT\bY \]

xxt <- torch_tensor(xx)
yt <- torch_tensor(y)

xxt$t()$matmul(xxt)$inverse()$matmul(xxt$t())$matmul(yt)
#> torch_tensor
#>   3.2734
#>  12.2067
#>  -9.5942
#> [ CPUFloatType{3,1} ]