MXB106 Lecture Notes.tex

%!TEX TS-program = xelatex
%!TEX options = -aux-directory=Debug -shell-escape -file-line-error -interaction=nonstopmode -halt-on-error -synctex=1 "%DOC%"
\documentclass{article}
\input{LaTeX-Submodule/template.tex}

% Additional packages & macros

% Header and footer
\newcommand{\unitName}{Linear Algebra}
\newcommand{\unitTime}{Semester 1, 2021}
\newcommand{\unitCoordinator}{Dr Ravindra Pethiyagoda}
\newcommand{\documentAuthors}{\textsc{Tarang Janawalkar}}

\fancyhead[L]{\unitName}
\fancyhead[R]{\leftmark}
\fancyfoot[C]{\thepage}

% Copyright
\usepackage[
    type={CC},
    modifier={by-nc-sa},
    version={4.0},
    imagewidth={5em},
    hyphenation={raggedright}
]{doclicense}

\date{}

\begin{document}
%
\begin{titlepage}
    \vspace*{\fill}
    \begin{center}
        \LARGE{\textbf{\unitName}} \\[0.1in]
        \normalsize{\unitTime} \\[0.2in]
        \normalsize\textit{\unitCoordinator} \\[0.2in]
        \documentAuthors
    \end{center}
    \vspace*{\fill}
    \doclicenseThis
    \thispagestyle{empty}
\end{titlepage}
\newpage
%
\tableofcontents
\newpage
%
\section{Euclidean Vector Spaces}
\subsection{Vectors}
\begin{definition}
    An \(n\)-dimensional \textbf{vector} is an ordered list of \(n\)
    numbers.
    \begin{equation*}
        \symbf{v} =
        \begin{bmatrix}
            v_1    \\
            v_2    \\
            \vdots \\
            v_n
        \end{bmatrix}
        \in \R^n
    \end{equation*}
\end{definition}
\begin{theorem}
    \(\R^n\) is the set of all ordered \(n\)-tuples of real
    numbers.
    \begin{equation*}
        \R^n = \bigl\{\left(v_1,\: v_2,\: \ldots,\: v_n\right): v_1,\: v_2,\: \ldots,\: v_n \in \R: n \in \N\bigr\}
    \end{equation*}
\end{theorem}
\underline{Notation:}
\begin{enumerate}
    \item Component form: \(\symbf{v} = \abracket*{v_1,\:
          v_2} = \left(v_1,\: v_2\right) =
          \begin{pmatrix}
              v_1 \\
              v_2
          \end{pmatrix}
          =
          \begin{bmatrix}
              v_1 \\
              v_2
          \end{bmatrix}
          \)
    \item Unit vector form: \(\symbf{v} = v_1\hat{\symbf{i}} +
          v_2\hat{\symbf{j}}\), where \(\hat{\symbf{i}}\) and
          \(\hat{\symbf{j}}\) are basis vectors along the \(x\)
          and \(y\) axes respectively.
    \item Denotation: \(\symbf{v} = \underset{\sim}{v} = \vec{v}\)
\end{enumerate}
\subsection{Position and Displacement Vectors}
\begin{definition}
    The \textbf{displacement vector} \(\overrightarrow{AB}\) from
    \(\symbf{a}\) to \(\symbf{b}\) can be defined as
    \(\symbf{b}-\symbf{a}\).
    \begin{figure}[H]
        \centering
        \includegraphics*{figures/vector_position.pdf}
        \caption{Displacement vector between two points.}
    \end{figure}
\end{definition}
\subsection{Vector Addition}
\begin{definition}
    \textbf{Vector addition} is performed by adding the corresponding
    components of two vectors of the same dimension.
    \begin{equation*}
        \symbf{a} + \symbf{b} =
        \begin{bmatrix}
            a_1+b_1 \\
            \vdots  \\
            a_n+b_n
        \end{bmatrix}
    \end{equation*}
\end{definition}
\subsection{Scalar Multiplication}
\begin{definition}
    \textbf{Scalar multiplication} is performed by multiplying each
    element of the vector by the scalar.
    \begin{equation*}
        a\symbf{v} =
        \begin{bmatrix}
            av_1   \\
            \vdots \\
            av_n
        \end{bmatrix}
    \end{equation*}
\end{definition}
\subsection{Norm of a Vector}
\begin{definition}
    The \textbf{norm} of a vector \(\symbf{v}\), denoted by
    \(\norm{\symbf{v}}\), is the \textit{length} or \textit{magnitude}
    of \(\symbf{v}\).
    \begin{equation*}
        \norm{\symbf{v}} = \sqrt{v_1^2 + v_2^2+\cdots+v_n^2}
    \end{equation*}
\end{definition}
\subsection{The Unit Vector}
\begin{definition}
    A \textbf{unit vector} is a vector, denoted \(\symbf{\hat{v}}\),
    that has a length of 1 in the direction of \(\symbf{v}\).
    \begin{equation*}
        \symbf{\hat{v}} = \frac{\symbf{v}}{\norm{\symbf{v}}}
    \end{equation*}
\end{definition}
\subsection{The Dot Product}
\begin{definition}
    The \textbf{dot product} is a function that associates each pair of
    vectors \(\symbf{v},\: \symbf{w} \in \R^n\) a real
    number \(\symbf{v}\cdot\symbf{w}\).
    \begin{align*}
        \symbf{v}\cdot\symbf{w} & = v_1 w_1 + v_2 w_2 + \cdots + v_n w_n                        \\
                                & = \norm{\symbf{v}} \norm{\symbf{w}} \cos{\left(\theta\right)}
    \end{align*}
    where \(\theta\) is the angle between \(\symbf{v}\) and
    \(\symbf{w}\).
\end{definition}
\begin{theorem}
    If \(\symbf{v} \cdot \symbf{w}=0\) then \(\symbf{v}\) and
    \(\symbf{w}\) are orthogonal.
\end{theorem}
\subsection{The Cross Product}
\begin{definition}
    The \textbf{cross product} is a function that associates each
    ordered pair of vectors
    \(\symbf{v},\: \symbf{w}\in \R^3\) a vector
    \(\symbf{v}\times\symbf{w}\in \R^3\).
    \begin{align*}
        \symbf{v}\times\symbf{w} & =
        \begin{vmatrix}
            \symbf{\hat{\imath}} & \symbf{\hat{\jmath}} & \symbf{\hat{k}} \\
            v_1                  & v_2                  & v_3             \\
            w_1                  & w_2                  & w_3
        \end{vmatrix}
        \\
                                 & = \norm{\symbf{v}}\norm{\symbf{w}}\sin{\left(\theta\right)}\symbf{\hat{n}}
    \end{align*}
    where \(\symbf{\hat{n}}\) is the normal vector given by the
    right-hand rule.
\end{definition}
\section{Vector Identities}
Let \(\symbf{a}, \symbf{b}, \symbf{c} \in V\) with \(r \in \R\).
\begin{theorem}
    Commutativity of vector addition.
    \begin{equation*}
        \symbf{a} + \symbf{b} = \symbf{b} + \symbf{a}
    \end{equation*}
\end{theorem}
\begin{theorem}
    \begin{equation*}
        \symbf{a}\cdot\symbf{a}=\norm{\symbf{a}}^2
    \end{equation*}
\end{theorem}
\begin{theorem}
    Commutativity of dot products.
    \begin{equation*}
        \symbf{a}\cdot \symbf{b} = \symbf{b}\cdot \symbf{a}
    \end{equation*}
\end{theorem}
\begin{theorem}
    Distributivity of dot products over vector addition.
    \begin{equation*}
        \symbf{a}\cdot \left(\symbf{b}+\symbf{c}\right) = \symbf{a}\cdot\symbf{b} + \symbf{a}\cdot\symbf{c}
    \end{equation*}
\end{theorem}
\begin{theorem}
    Associativity of dot products over scalar multiplication.
    \begin{equation*}
        \left(r\symbf{a}\right)\cdot\symbf{b} = r\left(\symbf{a}\cdot\symbf{b}\right)=\symbf{a}\cdot\left(r\symbf{b}\right)
    \end{equation*}
\end{theorem}
\begin{theorem}
    Bilinearity of dot products.
    \begin{equation*}
        \symbf{a}\cdot\left(r\symbf{b}+\symbf{c}\right)=r\left(\symbf{a}\cdot\symbf{b}\right)+\left(\symbf{a}\cdot\symbf{c}\right)
    \end{equation*}
\end{theorem}
\begin{theorem}
    \begin{equation*}
        \symbf{a}\times \symbf{a} = \symbf{0}
    \end{equation*}
\end{theorem}
\begin{theorem}
    Anti-commutativity of cross products.
    \begin{equation*}
        \symbf{a}\times \symbf{b} = -\symbf{b}\times \symbf{a}
    \end{equation*}
\end{theorem}
\begin{theorem}
    Distributivity of cross products over vector addition.
    \begin{equation*}
        \symbf{a}\times\left( \symbf{b}+\symbf{c} \right) = \symbf{a}\times\symbf{b} + \symbf{a}\times\symbf{c}
    \end{equation*}
\end{theorem}
\begin{theorem}
    Associativity of cross products over scalar multiplication.
    \begin{equation*}
        \left( r \symbf{a}\right)\times\symbf{b} = \symbf{a}\times\left( r \symbf{b} \right) = r\left(\symbf{a}\times\symbf{b}\right)
    \end{equation*}
\end{theorem}
\begin{theorem}
    \begin{equation*}
        \symbf{a} \cdot \left(\symbf{b}\times\symbf{c}\right) = \symbf{b} \cdot \left(\symbf{c}\times\symbf{a}\right) = \symbf{c} \cdot \left(\symbf{a}\times\symbf{b}\right)
    \end{equation*}
\end{theorem}
\begin{theorem}
    \begin{equation*}
        \symbf{a} \times \left(\symbf{b}\times\symbf{c}\right) = \symbf{b} \left(\symbf{a}\cdot\symbf{c}\right) - \symbf{c}\left(\symbf{a}\cdot\symbf{b}\right)
    \end{equation*}
\end{theorem}
\section{Linear System of Equations}
\subsection{Linear Equations}
\begin{definition}
    A \textbf{linear equation} in \(n\) variables
    \(x_1,\: x_2,\: \dots, x_n\) can be expressed in the form
    \begin{equation*}
        a_{1}x_1 + a_{2}x_2 + \cdots + a_{n}x_n = b
    \end{equation*}
    where the \textit{coefficients} \(a_1,\: a_2,\: \dots, a_n\) and the
    \textit{constant term} \(b\) are constants.
\end{definition}
\subsection{Homogeneous Linear Equations}
\begin{definition}
    In the special case where \(b=0\), the linear equation is called a
    \textbf{homogeneous linear equation}.
\end{definition}
\subsection{Linear Systems}
\begin{definition}
    A \textbf{linear system of equations} is a set of linear equations,
    where the variables \(x_i\) are called \textit{unknowns}. The
    general linear system of \(m\) equations with \(n\) unknowns can be
    written as
    \begin{equation*}
        \left\{
        \setlength\arraycolsep{0pt}
        \begin{array}
            { c >{{}}c<{{}} c >{{}}c<{{}} c >{{}}c<{{}} c >{{}}c<{{}} c }
            a_{11}x_1                         & + & a_{12}x_2                         & + & \cdots & + & a_{1n}x_n                         & = & b_1    \\
            a_{21}x_1                         & + & a_{22}x_2                         & + & \cdots & + & a_{2n}x_n                         & = & b_2    \\
            \vdotswithin{a_{31}}\phantom{x_1} &   & \vdotswithin{a_{32}}\phantom{x_2} &   &        &   & \vdotswithin{a_{3n}}\phantom{x_n} &   & \vdots \\
            a_{m1}x_1                         & + & a_{m2}x_2                         & + & \cdots & + & a_{mn}x_n                         & = & b_m
        \end{array}
        \right.
    \end{equation*}
    A \textbf{solution} to the system is an \(n\)-tuple
    \(\left\langle x_1,\: x_2,\:\dots,\:x_n\right\rangle\) that
    satisfies each equation.
\end{definition}
\subsection{Coefficient Matrices}
\begin{definition}
    The coefficients of the variables in each equation can be placed
    inside the system's \textbf{coefficient matrix}.
    \begin{equation*}
        \begin{bmatrix}
            a_{11} & a_{12} & \cdots & a_{1n} \\
            a_{21} & a_{22} & \cdots & a_{2n} \\
            \vdots & \vdots &        & \vdots \\
            a_{m1} & a_{m2} & \cdots & a_{mn}
        \end{bmatrix}
    \end{equation*}
\end{definition}
\subsection{Augmented Matrices}
\begin{definition}
    The information of a system can be contained in its
    \textbf{augmented matrix}.
    \begin{equation*}
        \begin{bmatrix}[cccc|c]
            a_{11} & a_{12} & \cdots & a_{1n} & b_1    \\
            a_{21} & a_{22} & \cdots & a_{2n} & b_2    \\
            \vdots & \vdots &        & \vdots & \vdots \\
            a_{m1} & a_{m2} & \cdots & a_{mn} & b_m
        \end{bmatrix}
    \end{equation*}
\end{definition}
\begin{definition}
    An array having \(m\) rows and \(n\) columns, is an \(m \times n\)
    \textbf{matrix}. This matrix may be denoted as \(a_{ij}\), where
    \(a_{ij}\) is the entry in \(i\)th row and \(j\)th column of the
    matrix \(\symbf{A}\).
    \begin{equation*}
        \text{\(m\) rows}
        \left\{
        \left[
        \vphantom{
            \begin{array}{c} 1 \\ 1 \\ 1 \\ 1
            \end{array}
        }
        \smash{\underbrace{
            \begin{array}{cccc}
                a_{11} & a_{12} & \cdots & a_{1n} \\
                a_{21} & a_{22} & \cdots & a_{2n} \\
                \vdots & \vdots &        & \vdots \\
                a_{m1} & a_{m2} & \cdots & a_{mn}
            \end{array}
        }_{\text{\(n\) columns}}}
        \right]
        \right.
    \end{equation*}
\end{definition}
\subsection{Elementary Row Operations}
\begin{definition}
    A linear system can be solved using the following
    \textbf{elementary row operations}:
    \begin{enumerate}
        \item \textbf{scalar multiplication}: multiplying any row by a
              constant
        \item \textbf{row addition}: adding a multiple of one row to
              another
        \item \textbf{row exchange}: exchanging any two rows
    \end{enumerate}
    \subsection{Pivots}
\end{definition}
\begin{definition}
    The first non-zero entry of the row in a matrix is called the
    \textbf{pivot} of the row.
\end{definition}
\begin{theorem}
    \label{theorem:pivots}
    If a row apart from the first has a pivot, then this pivot must be
    to the \textit{right} of the pivot in the preceding row.
\end{theorem}
\subsection{Gaussian Elimination}
\begin{definition}
    \textbf{Gaussian elimination} is a method for solving linear
    systems. These systems can be solved by composing the augmented
    matrix of a system, and performing elementary row operations, to put
    the matrix in the form
    \begin{equation*}
        \begin{bmatrix}
            a_{11} & a_{12} & \cdots & a_{1n} \\
                   & a_{22} & \cdots & a_{2n} \\
                   &        & \ddots & \vdots \\
                   &        &        & a_{mn}
        \end{bmatrix}
    \end{equation*}
\end{definition}
\subsection{Row-Echelon Form}
\begin{definition}
    A matrix that has undergone Gaussian elimination is in
    \textbf{row-echelon form} if the pivots of the augmented matrix are
    all 1.
    \begin{equation*}
        \begin{bmatrix}
            1 & a_{12} & \cdots & a_{1n} \\
              & 1      & \cdots & a_{2n} \\
              &        & \ddots & \vdots \\
              &        &        & 1
        \end{bmatrix}
    \end{equation*}
\end{definition}
\subsection{Gauss-Jordan Elimination}
\begin{definition}
    \textbf{Gauss-Jordan elimination} extends Gaussian elimination so
    that the entries in a column containing a pivot are zeros, and the
    pivots are all 1. This new augmented matrix is then in
    \textbf{reduced row-echelon form}.
    \begin{equation*}
        \begin{bmatrix}
            1 &   &        &   \\
              & 1 &        &   \\
              &   & \ddots &   \\
              &   &        & 1
        \end{bmatrix}
    \end{equation*}
\end{definition}
\subsection{Solutions to Linear Systems}
\begin{definition}
    A \textbf{consistent system} of equations has at least one solution,
    and an \linebreak \textbf{inconsistent system} has no solution.
\end{definition}
\section{Matrices}
\begin{definition}
    A \textbf{matrix} is an array of numbers arranged into \textit{rows}
    and \textit{columns}, and can be used to represent a linear
    transformation.
    \begin{equation*}
        \symbf{A} =
        \begin{bmatrix}
            a_{11} & a_{12} & \cdots & a_{1n} \\
            a_{21} & a_{22} & \cdots & a_{2n} \\
            \vdots & \vdots &        & \vdots \\
            a_{m1} & a_{m2} & \cdots & a_{mn} \\
        \end{bmatrix}
        \in \R^{m \times n}
    \end{equation*}
\end{definition}
\subsection{Matrix Addition}
\begin{definition}
    \textbf{Matrix addition} is performed by adding the corresponding
    components of two matrices of the same dimension.
    \begin{equation*}
        \symbf{A} + \symbf{B} =
        \begin{bmatrix}
            a_{11} + b_{11} & a_{12} + b_{12} & \cdots & a_{1n} + b_{1n} \\
            a_{21} + b_{21} & a_{22} + b_{22} & \cdots & a_{2n} + b_{2n} \\
            \vdots          & \vdots          &        & \vdots          \\
            a_{m1} + b_{m1} & a_{m2} + b_{m2} & \cdots & a_{mn} + b_{mn} \\
        \end{bmatrix}
    \end{equation*}
\end{definition}
\subsection{Scalar Multiplication}
\begin{definition}
    \textbf{Scalar multiplication} is performed by multiplying each
    element of a matrix by a scalar.
    \begin{equation*}
        c\symbf{A} =
        \begin{bmatrix}
            ca_{11} & ca_{12} & \cdots & ca_{1n} \\
            ca_{21} & ca_{22} & \cdots & ca_{2n} \\
            \vdots  & \vdots  &        & \vdots  \\
            ca_{m1} & ca_{m2} & \cdots & ca_{mn} \\
        \end{bmatrix}
    \end{equation*}
\end{definition}
\subsection{Matrix Multiplication}
\begin{definition}
    \textbf{Matrix multiplication} is performed by multiplying each row
    in the first matrix by the columns of the second matrix.
    \begin{align*}
        \symbf{A} \symbf{B} & = \symbf{C} \\
        \begin{bmatrix}
            \horzbar & \symbf{a}_{1} & \horzbar \\
            \horzbar & \symbf{a}_{2} & \horzbar \\
                     & \vdots        &          \\
            \horzbar & \symbf{a}_{m} & \horzbar
        \end{bmatrix}
        \begin{bmatrix}
            \vertbar      & \vertbar      &        & \vertbar      \\
            \symbf{b}_{1} & \symbf{b}_{2} & \cdots & \symbf{b}_{n} \\
            \vertbar      & \vertbar      &        & \vertbar
        \end{bmatrix}
                            & =
        \begin{bmatrix}
            \symbf{a}_1\symbf{b}_1 & \symbf{a}_1\symbf{b}_2 & \cdots & \symbf{a}_1\symbf{b}_n \\
            \symbf{a}_2\symbf{b}_1 & \symbf{a}_2\symbf{b}_2 & \cdots & \symbf{a}_2\symbf{b}_n \\
            \vdots                 & \vdots                 &        & \vdots                 \\
            \symbf{a}_m\symbf{b}_1 & \symbf{a}_m\symbf{b}_2 & \cdots & \symbf{a}_m\symbf{b}_n
        \end{bmatrix}
    \end{align*}
\end{definition}
\begin{theorem}
    A matrix product is defined if and only if the number of columns in
    the first matrix is equal to the number of rows in the second
    matrix.
\end{theorem}
\subsection{The Identity Matrix}
\begin{definition}
    The \textbf{identity matrix} is the simplest nontrivial
    \textbf{diagonal matrix}, denoted \(\symbf{I}\), such that
    \begin{equation*}
        \symbf{I} \symbf{A} = \symbf{A}
    \end{equation*}
    written explicitly as
    \begin{equation*}
        \symbf{I} =
        \begin{bmatrix}
            1 &   &        &   \\
              & 1 &        &   \\
              &   & \ddots &   \\
              &   &        & 1
        \end{bmatrix}
    \end{equation*}
\end{definition}
\subsection{The Inverse Matrix}
\begin{definition}
    The \textbf{inverse} of a \textbf{square matrix} is a matrix
    \(\symbf{A}^{-1}\), such that
    \begin{equation*}
        \symbf{A} \symbf{A}^{-1} = \symbf{I}
    \end{equation*}
\end{definition}
\begin{theorem}
    The inverse of a \(2\times 2\) matrix is given by
    \begin{equation*}
        \symbf{A}^{-1} = \frac{1}{ad-bc}
        \begin{bmatrix}
            a_{22}  & -a_{12} \\
            -a_{21} & a_{11}
        \end{bmatrix}
    \end{equation*}
\end{theorem}
\begin{theorem}
    The inverse of an \(n\times n\) matrix can be determined by solving
    \(
    \begin{bmatrix}[c|c]
        \symbf{A} & \symbf{I}
    \end{bmatrix}
    \).
\end{theorem}
\subsection{The Diagonal Matrix}
\begin{definition}
    A \textbf{diagonal} matrix, denoted
    \(\diag{\left( d_{11},\: d_{22},\: \ldots,\: d_{nn} \right)}\), is
    an \(n \times n\) matrix \(\symbf{D}\) in which entries outside
    the main diagonal are all zero.
    \begin{equation*}
        \symbf{D} = \diag{\left( d_{11}, d_{22}, \ldots, d_{nn} \right)} =
        \begin{bmatrix}
            d_{11} &        &        &        \\
                   & d_{22} &        &        \\
                   &        & \ddots &        \\
                   &        &        & d_{nn}
        \end{bmatrix}
    \end{equation*}
\end{definition}
\subsection{Matrix Transpose}
\begin{definition}
    The \textbf{transpose} of a matrix, denoted by \(\symbf{A}^\top\),
    is obtained by replacing all \(a_{ij}\) elements with \(a_{ji}\), so
    that the matrix \(\symbf{A}\) is flipped over its main diagonal.
\end{definition}
\subsection{Matrix Trace}
\begin{definition}
    The \textbf{trace} of an \(n \times n\) matrix \(\symbf{A}\),
    denoted \(\Tr{\left( \symbf{A} \right)}\), is defined as
    \begin{equation*}
        \Tr{\left( \symbf{A} \right)} = \sum_{i=1}^n a_{ii}
    \end{equation*}
\end{definition}
\section{General Vector Spaces}
\subsection{Real Vector Spaces}
\begin{definition}
    A \textbf{vector space} is a set that is closed under vector
    addition and scalar \linebreak multiplication.
\end{definition}
\begin{theorem}
    If the following axioms are satisfied by all objects
    \(\symbf{u},\: \symbf{v},\: \symbf{w} \in V\), and all scalars
    \(k\) and \(m\), then \(V\) is a \textbf{vector space}, and the
    objects in \(V\) are vectors.
\end{theorem}
\begin{axiom}[Closure under addition]
    \begin{equation*}
        \label{axiom:5_1_1}
        \symbf{u}+\symbf{v} \in V
    \end{equation*}
\end{axiom}
\begin{axiom}[Commutativity of vector addition]
    \begin{equation*}
        \label{axiom:5_1_2}
        \symbf{u} + \symbf{v} = \symbf{v} + \symbf{u}
    \end{equation*}
\end{axiom}
\begin{axiom}[Associativity of vector addition]
    \begin{equation*}
        \label{axiom:5_1_3}
        \symbf{u} + \left(\symbf{v} + \symbf{w}\right) = \left(\symbf{u} + \symbf{v}\right) + \symbf{w}
    \end{equation*}
\end{axiom}
\begin{axiom}[Additive identity]
    \begin{equation*}
        \label{axiom:5_1_4}
        \symbf{u} + \symbf{0} = \symbf{u}
    \end{equation*}
\end{axiom}
\begin{axiom}[Additive inverse]
    \begin{equation*}
        \label{axiom:5_1_5}
        \symbf{u} + \left(-\symbf{u}\right) = \symbf{0}
    \end{equation*}
\end{axiom}
\begin{axiom}[Closure under scalar multiplication]
    \begin{equation*}
        \label{axiom:5_1_6}
        k\symbf{u} \in V
    \end{equation*}
\end{axiom}
\begin{axiom}[Distributivity of vector addition]
    \begin{equation*}
        \label{axiom:5_1_7}
        k \left(\symbf{u} + \symbf{v}\right) = k\symbf{u} + k\symbf{v}
    \end{equation*}
\end{axiom}
\begin{axiom}[Distributivity of scalar addition]
    \begin{equation*}
        \label{axiom:5_1_8}
        \left(k+m\right)\symbf{u} = k\symbf{u} + m\symbf{u}
    \end{equation*}
\end{axiom}
\begin{axiom}[Associativity of scalar multiplication]
    \begin{equation*}
        \label{axiom:5_1_9}
        k\left(m\symbf{u}\right)=\left(km\right)\symbf{u}
    \end{equation*}
\end{axiom}
\begin{axiom}[Scalar multiplication identity]
    \begin{equation*}
        \label{axiom:5_1_10}
        1 \symbf{u}=\symbf{u}
    \end{equation*}
\end{axiom}
\noindent To identify that a set with two operations is a vector space:
\begin{enumerate}
    \item Identify the set \(V\) of objects that will become vectors.
    \item Identify the addition and scalar multiplication operations on
          \(V\).
    \item Verify Axioms~\ref{axiom:5_1_1} and~\ref{axiom:5_1_6}.
    \item Confirm that
          Axioms~\ref{axiom:5_1_2},~\ref{axiom:5_1_3},~\ref{axiom:5_1_4},~\ref{axiom:5_1_5},~\ref{axiom:5_1_7},~\ref{axiom:5_1_8},~\ref{axiom:5_1_9},
          and~\ref{axiom:5_1_10} hold.
\end{enumerate}
\begin{theorem}
    Let \(V\) be a vector space. If \(\symbf{v} \in V\), and \(k\) is
    a scalar.
    \begin{enumerate}
        \item \(0\symbf{v}=\symbf{0}\)
        \item \(k\symbf{0}=\symbf{0}\)
        \item \(\left(-1\right)\symbf{v}=-\symbf{v}\)
        \item If \(k\symbf{v}=\symbf{0}\), then \(k=0\) or
              \(\symbf{v}=\symbf{0}\)
    \end{enumerate}
\end{theorem}
\subsection{Subspaces}
\begin{definition}
    A \textit{subset} \(W\) of a vector space \(V\) is called a
    \textbf{subspace} of \(V\) if \(W\) is itself a vector space under
    the addition and scalar multiplication operations defined on \(V\).
\end{definition}
\begin{theorem}
    Let \(W\) be a subspace of the vector space \(V\), then the
    following axioms must be satisfied.
    \begin{enumerate}
        \item \textbf{\hyperref[axiom:5_1_1]{Axiom~\ref{axiom:5_1_1}}}:
              Closure under addition
        \item \textbf{\hyperref[axiom:5_1_6]{Axiom~\ref{axiom:5_1_6}}}:
              Closure under scalar multiplication
    \end{enumerate}
\end{theorem}
\begin{theorem}
    Every vector space has at least two subspaces, itself and its zero
    subspace.
\end{theorem}
\begin{theorem}
    Subspaces of \(\R^2\).
    \begin{enumerate}
        \item \(\left\{ \symbf{0} \right\}\)
        \item Lines through the origin
        \item \(\R^2\)
    \end{enumerate}
\end{theorem}
\begin{theorem}
    Subspaces of \(\R^3\).
    \begin{enumerate}
        \item \(\left\{ \symbf{0} \right\}\)
        \item Lines through the origin
        \item Planes through the origin
        \item \(\R^3\)
    \end{enumerate}
\end{theorem}
\begin{theorem}
    Subspaces of \(\symbf{M}_{nn}\).
    \begin{enumerate}
        \item Upper triangular matrices
        \item Lower triangular matrices
        \item Diagonal matrices
        \item \(\symbf{M}_{nn}\)
    \end{enumerate}
\end{theorem}
\subsection{Spanning Sets}
\begin{definition}
    If the vector \(\symbf{w}\) is in a vector space \(V\), then
    \(\symbf{w}\) is a \textbf{linear combination} of the vectors
    \(\symbf{v}_1,\: \symbf{v}_2,\: \dots,\: \symbf{v}_n \in V\),
    if \(\symbf{w}\) can be expressed in the form
    \begin{equation*}
        \symbf{w} = k_1 \symbf{v}_1 + k_2 \symbf{v}_2 + \cdots + k_n \symbf{v}_n
    \end{equation*}
\end{definition}
\begin{theorem}
    If \(S=\left\{ \symbf{w}_1,\: \symbf{w}_2,\: \dots,\: \symbf{w}_n \right\}\)
    is a nonempty set of vectors in a vector space \(V\), then the set
    \(W\) of all possible linear combinations of the vectors in \(S\) is
    a subspace of \(V\). The subspace \(W\) is called the subspace of
    \(V\) \textbf{spanned} by \(S\) and the vectors in \(S\)
    \textbf{span} \(W\). If a vector in \(S\) can be expressed as the
    linear combination of any vectors in \(S\) then the set is
    \textbf{linearly dependent}.
\end{theorem}
\subsection{Linear Independence}
\begin{definition}
    If \(S\) is a set of two or more vectors in a vector space \(V\),
    then \(S\) is \textbf{linearly independent} if no vector in \(S\)
    can be expressed as a linear combination of the others.
\end{definition}
\begin{theorem}
    A set \(S\) is linearly independent if and only if there is one
    solution to the equation
    \begin{equation*}
        k_1 \symbf{v}_1 + k_2 \symbf{v}_2 + \cdots + k_n \symbf{v}_n = \symbf{0}
    \end{equation*}
    where the coefficients satisfying this equation are
    \(k_1=0, k_2=0, \dots, k_n=0\).
\end{theorem}
\subsection{Basis Vectors}
\begin{definition}
    If \(S\) is a set of vectors in a vector space \(V\), then \(S\) is
    called a \textbf{basis} for \(V\) if
    \begin{enumerate}
        \item \(S\) spans \(V\).
        \item \(S\) is linearly independent.
    \end{enumerate}
\end{definition}
\subsection{Dimension}
\begin{definition}
    The \textbf{dimension} of a finite-dimensional vector space \(V\),
    denoted \(\dim{\left( V \right)}\), is the number of vectors in a
    basis for \(V\).
\end{definition}
\begin{theorem}
    The zero vector space is defined to have dimension zero.
\end{theorem}
\section{Fundamental Subspaces}
\begin{figure}[H]
    \centering
    \includegraphics[height=10cm, keepaspectratio]{figures/fundamental_subspaces.pdf}
    \caption{The Four Fundamental Subspaces of a Matrix.}
\end{figure}
\subsection{The Four Fundamental Subspaces of a Matrix}
\begin{definition}
    If \(\symbf{A}\in\R^{m \times n}\) is an \(m \times n\) matrix, then:
    \begin{enumerate}
        \item The subspace spanned by the \textit{column vectors} of
              \(\symbf{A}\), is the \textbf{column space} of
              \(\symbf{A}\), denoted \(\columnspace{A}\).
        \item The subspace spanned by the \textit{row vectors} of
              \(\symbf{A}\), is the \textbf{row space} of
              \(\symbf{A}\), denoted \(\rowspace{A}\).
        \item The subspace spanned by the \textit{solution space} of
              the equation \(\symbf{A}\symbf{x}=\symbf{0}\), is the
              \textbf{null space} of \(\symbf{A}\), denoted
              \(\nullspace{A}\).
        \item The subspace spanned by the \textit{solution space} of
              the equation \(\symbf{A}^{\top}\symbf{y}=\symbf{0}\) (or
              \(\symbf{y}^{\top}\symbf{A}=\symbf{0}\)), is the
              \textbf{left null space} of \(\symbf{A}\), denoted
              \(\leftnullspace{A}\).
    \end{enumerate}
\end{definition}
\subsection{The General Solution of a System of Equations}
\begin{theorem}
    The \textbf{general solution} to a matrix equation
    \(\symbf{A}\symbf{x}=\symbf{b}\), can be given by adding the
    \underline{particular} and \underline{homogeneous} solutions, where
    the particular solution is the solution to
    \(\symbf{A}\symbf{x}=\symbf{b}\), or \(\rowspace{A}\), and the
    homogeneous solution is the solution to
    \(\symbf{A}\symbf{x}=\symbf{0}\), or \(\nullspace{A}\).
    \begin{equation*}
        \symbf{x} = \symbf{x}_p + \symbf{x}_h
    \end{equation*}
\end{theorem}
\subsection{Row Equivalence}
\begin{definition}
    Two matrices are \textbf{row equivalent} if each can be obtained
    from the other by elementary row operations. These matrices have the
    same \underline{row space} and \underline{null space}.
\end{definition}
\subsection{Rank}
\begin{definition}
    The \textbf{rank} of a matrix, denoted by
    \(\vrank{\left( \symbf{A} \right)}\), is given by
    \(\dim{\left( \columnspace{A} \right)}\).
\end{definition}
\begin{theorem}
    The column space and row space have the same dimension so that
    \begin{equation*}
        \vrank{\left( \symbf{A} \right)}=\dim{\left( \columnspace{A} \right)}=\dim{\left( \rowspace{A} \right)}
    \end{equation*}
\end{theorem}
\subsection{Nullity}
\begin{definition}
    The \textbf{nullity} of a matrix, denoted by
    \(\vnull{\left( \symbf{A} \right)}\), is given by
    \(\dim{\left( \nullspace{A} \right)}\).
\end{definition}
\section{Orthogonality}
\begin{definition}
    Two vectors are \textbf{orthogonal} if the following holds.
    \begin{equation*}
        \symbf{v}\cdot \symbf{w}=0 \iff \symbf{v}^{\top}\symbf{w}=0
    \end{equation*}
\end{definition}
\begin{theorem}
    \(\symbf{0}\) is orthogonal to every vector in \(V\).
\end{theorem}
\begin{theorem}
    \(\symbf{0}\) is the only vector in \(V\), that is orthogonal to
    itself.
\end{theorem}
\begin{theorem}
    \begin{equation*}
        \norm{\symbf{v}}^2=\symbf{v}^{\top}\symbf{v}
    \end{equation*}
\end{theorem}
\begin{theorem}
    \begin{equation*}
        \norm{\symbf{v}}=\sqrt{\symbf{v}^{\top}\symbf{v}}
    \end{equation*}
\end{theorem}
\subsection{Orthogonal Subspaces}
\begin{definition}
    Two subspaces \(U\) and \(W\) of a vector space \(V\), are
    \textbf{orthogonal subspaces} iff every vector in \(U\) is
    orthogonal to every vector in \(W\).
    \begin{equation*}
        \forall \symbf{u}\in U:\forall \symbf{w}\in W:\symbf{u}^{\top}\symbf{w}=0
    \end{equation*}
\end{definition}
\subsection{Orthogonal Complements}
\begin{definition}
    If \(U\) is a subspace of \(V\), then the
    \textbf{orthogonal complement} of \(U\), denoted \(U^{\perp}\), is
    the set of all vectors in \(V\) that are orthogonal to every vector
    in \(U\).
    \begin{equation*}
        U^{\perp} = \left\{ \forall \symbf{u}\in U:\symbf{v}\in V: \symbf{v}^{\top}\symbf{u}=0 \right\}
    \end{equation*}
\end{definition}
\begin{theorem}
    \begin{equation*}
        \left( U^{\perp} \right)^{\perp}=U
    \end{equation*}
\end{theorem}
\begin{theorem}
    \begin{equation*}
        \dim{U} + \dim{U^{\perp}} = \dim{V}
    \end{equation*}
\end{theorem}
\subsection{Vector Projections}
\begin{figure}[H]
    \centering
    \includegraphics[height=5cm, keepaspectratio]{figures/vector_projection.pdf}
    \caption{Vector Projection of \(\symbf{b}\) onto \(\symbf{a}\).}
\end{figure}
\begin{definition}
    Let the \textbf{vector projection} of \(\symbf{b}\) onto
    \(\symbf{a}\), denoted as \(\proj_{\symbf{a}}\symbf{b}\), be
    the \textit{orthogonal projection} of \(\symbf{b}\)
    in the direction of \(\symbf{a}\), that minimises the error
    vector: \(\symbf{e}=\symbf{b}-\symbf{p}\).
\end{definition}
\begin{theorem}
    The projection of \(\symbf{b}\) onto \(\symbf{a}\) is given by
    \begin{equation*}
        \proj_{\symbf{a}}\symbf{b} = \symbf{a} x = \symbf{a} \left( \symbf{a}^\top \symbf{a} \right)^{-1}\symbf{a}^\top \symbf{b}
    \end{equation*}
    alternatively
    \begin{equation*}
        \proj_{\symbf{a}}\symbf{b} = \symbf{a} x = \symbf{a} \frac{\symbf{a}^\top \symbf{b}}{\symbf{a}^\top \symbf{a}} = \symbf{a} \frac{\symbf{a}\cdot\symbf{b}}{\symbf{a}\cdot\symbf{a}}
    \end{equation*}
\end{theorem}
\begin{proof}
    As \(\symbf{p}\) lies on the line through
    \(\symbf{a}\), \(\symbf{p}=\symbf{a}x\), so that
    \(\symbf{e}=\symbf{b}-\symbf{a}x\). As \(\symbf{e}\) is
    orthogonal to \(\symbf{a}\), we can construct the following
    relationship.
    \begin{align*}
        \symbf{a}^\top \symbf{e}                             & = 0                                                                                                                                \\
        \symbf{a}^\top \left( \symbf{b} - \symbf{a}x \right) & = 0                                                                                                                                \\
        \symbf{a}^\top \symbf{b} - \symbf{a}^\top \symbf{a}x & = 0                                                                                                                                \\
        \symbf{a}^\top \symbf{a}x                            & = \symbf{a}^\top \symbf{b}                                                                                                         \\
        x                                                    & = \left( \symbf{a}^\top \symbf{a} \right)^{-1}\symbf{a}^\top \symbf{b} = \frac{\symbf{a}^\top \symbf{b}}{\symbf{a}^\top \symbf{a}}
    \end{align*}
\end{proof}
\subsection{Projection onto a Subspace}
\begin{theorem}
    Let \(W\) be a subspace of the vector space \(V\) such that if
    \(\symbf{b}\in V\), then \(\symbf{p}=\proj_{W}\symbf{b}\) is
    the \textbf{best approximation} of \(\symbf{b}\) on \(W\), so that
    \begin{equation*}
        \norm{\symbf{b}-\symbf{p}}<\norm{\symbf{b}-\symbf{w}}
    \end{equation*}
    for all \(\symbf{w}\in W\), where \(\symbf{w}\neq \symbf{p}\).
\end{theorem}
\begin{theorem}
    The projection of \(\symbf{b}\) onto the vector space \(W\) is
    given by
    \begin{equation*}
        \proj_{W}\symbf{b} = \symbf{A}\symbf{\hat{x}} = \symbf{A}\left( \symbf{A}^\top \symbf{A} \right)^{-1}\symbf{A}^\top \symbf{b}
    \end{equation*}
\end{theorem}
\begin{proof}
    As \(\symbf{p}\in W\), \(\symbf{p}\) can be represented as the
    linear combination of the basis vectors \(\symbf{a}_i\) that span
    \(W\).
    \begin{align*}
        \symbf{p} & = \hat{x}_1 \symbf{a}_1 + \hat{x}_2 \symbf{a}_2 + \cdots + \hat{x}_n \symbf{a}_n \\
                  & =
        \begin{bmatrix}
            \vertbar    & \vertbar    &        & \vertbar    \\
            \symbf{a}_1 & \symbf{a}_2 & \cdots & \symbf{a}_n \\
            \vertbar    & \vertbar    &        & \vertbar
        \end{bmatrix}
        \begin{bmatrix}
            \hat{x}_1 \\
            \hat{x}_2 \\
            \cdots    \\
            \hat{x}_n
        \end{bmatrix}
        \\
                  & = \symbf{A}\symbf{\hat{x}}
    \end{align*}
    Consider the error vector \(\symbf{e}=\symbf{b}-\symbf{p}\).
    As \(\symbf{e}\) is orthogonal to \(W\), it will also be
    orthogonal to the vectors that span \(W\). Therefore,
    \begin{equation*}
        \left\{
        \begin{aligned}
            \symbf{a}_1^\top\left( \symbf{b}-\symbf{A}\symbf{\hat{x}} \right)                         & = 0 \\
            \symbf{a}_2^\top\left( \symbf{b}-\symbf{A}\symbf{\hat{x}} \right)                         & = 0 \\
            \vdotswithin{\symbf{a}_3^\top}\phantom{\left( \symbf{b}-\symbf{A}\symbf{\hat{x}} \right)} &     \\
            \symbf{a}_n^\top\left( \symbf{b}-\symbf{A}\symbf{\hat{x}} \right)                         & = 0
        \end{aligned}
        \right.
    \end{equation*}
    which gives the following equation
    \begin{equation*}
        \symbf{A}^\top \left( \symbf{b}-\symbf{A}\symbf{\hat{x}} \right) = \symbf{0}
    \end{equation*}
    where we solve for \(\symbf{\hat{x}}\)
    \begin{align*}
        \symbf{A}^\top \symbf{b}-\symbf{A}^\top \symbf{A}\symbf{\hat{x}} & = \symbf{0}                                                            \\
        \symbf{A}^\top \symbf{A}\symbf{\hat{x}}                          & = \symbf{A}^\top \symbf{b}                                             \\
        \symbf{\hat{x}}                                                  & = \left( \symbf{A}^\top \symbf{A} \right)^{-1}\symbf{A}^\top \symbf{b}
    \end{align*}
\end{proof}
\subsection{Least Squares}
\begin{theorem}
    Suppose \(\symbf{A}\symbf{x}=\symbf{b}\) is an
    \underline{inconsistent} linear system. The \textbf{least squares}
    solution of \(\symbf{A}\symbf{x}=\symbf{b}\) is given by the
    orthogonal projection \(\proj_{\columnspace{A}}\symbf{b}\).
\end{theorem}
\section{Linear Maps}
\subsection{Matrix Transformations}
\begin{definition}
    A \textbf{matrix transformation}
    \(T_{\symbf{A}}:\R^n\rightarrow \R^m\) is a
    mapping of the form
    \begin{equation*}
        T_{\symbf{A}}\left(\symbf{x}\right) = \symbf{A}\symbf{x}
    \end{equation*}
    where \(\symbf{A}\in \R^{m \times n}\). As this
    transformation is linear, the following linearity properties hold.
    \begin{enumerate}
        \item \(T\left(\symbf{u}+\symbf{v}\right)=T\left(\symbf{u}\right) + T\left(\symbf{v}\right)\)
        \item \(T\left(k\symbf{u}\right)= kT\left(\symbf{u}\right)\)
    \end{enumerate}
\end{definition}
\subsection{General Linear Transformations}
\begin{theorem}
    If \(T: V \rightarrow W\) is a mapping between two vector spaces
    \(V\) and \(W\), then \(T\) is the \textbf{linear transformation}
    from \(V\) to \(W\), and the following properties hold.
    \begin{enumerate}
        \item \(T\left(\symbf{u}+\symbf{v}\right)=T\left(\symbf{u}\right) + T\left(\symbf{v}\right)\)
        \item \(T\left(k\symbf{u}\right) = kT\left(\symbf{u}\right)\)
    \end{enumerate}
\end{theorem}
\begin{theorem}
    When \(V=W\), the linear map is called a \textbf{linear operator}.
\end{theorem}
\subsection{Subspaces of Linear Transformations}
\begin{figure}[H]
    \centering
    \includegraphics[height=10cm, keepaspectratio]{figures/matrix_transformation.pdf}
    \caption{Subspaces of a Linear Transformation.}
\end{figure}
\begin{definition}
    If \(T:V \rightarrow W\) is a linear transformation between two
    vector spaces \(V\) and \(W\), then:
    \begin{enumerate}
        \item The vector space \(V\) is the \textbf{domain} of \(T\).
        \item The vector space \(W\) is the \textbf{codomain} of \(T\).
        \item The \textbf{image} (or \textbf{range}) of \(T\) is the
              set of vectors the linear transformation maps to.
              \begin{equation*}
                  \vim{\left( T \right)} = T\left(V\right) = \left\{ T\left(\symbf{v}\right) : \symbf{v}\in V \right\} \subset W
              \end{equation*}
        \item The \textbf{kernel} of \(T\) is the set of vectors that
              map to the zero vector.
              \begin{equation*}
                  \vker{\left( T \right)} = \left\{ \symbf{v}\in V : T\left(\symbf{v}\right)=\symbf{0} \right\}
              \end{equation*}
    \end{enumerate}
\end{definition}
\subsection{Constructing a Transformation Matrix}
\begin{theorem}
    The standard matrix for a linear transformation is given by the
    formula:
    \begin{equation*}
        \symbf{A} =
        \begin{bmatrix}
            \vertbar                  & \vertbar                  &        & \vertbar                  \\
            T\left(\symbf{e}_1\right) & T\left(\symbf{e}_2\right) & \cdots & T\left(\symbf{e}_n\right) \\
            \vertbar                  & \vertbar                  &        & \vertbar
        \end{bmatrix}
    \end{equation*}
    where
    \begin{equation*}
        \symbf{e}_1 =
        \begin{bmatrix}
            1      \\
            0      \\
            0      \\
            \vdots \\
            0
        \end{bmatrix}
        ,\, \symbf{e}_2 =
        \begin{bmatrix}
            0      \\
            1      \\
            0      \\
            \vdots \\
            0
        \end{bmatrix}
        ,\, \dots,\, \symbf{e}_n =
        \begin{bmatrix}
            0      \\
            0      \\
            0      \\
            \vdots \\
            1
        \end{bmatrix}
    \end{equation*}
    are the \textbf{standard basis vectors} for \(\R^n\).
\end{theorem}
\section{Determinants}
\subsection{Properties of Determinants}
\begin{enumerate}
    \item \(\det{\left( \symbf{I} \right)}=1\).
    \item Exchanging two rows of a matrix reverses the sign of its
          determinant.
    \item Determinants are multilinear, so that
          \begin{enumerate}[label=(\alph*)]
              \item \(
                    \begin{vmatrix}
                        a+a' & b+b' \\
                        c    & d
                    \end{vmatrix}
                    =
                    \begin{vmatrix}
                        a & b \\
                        c & d
                    \end{vmatrix}
                    +
                    \begin{vmatrix}
                        a' & b' \\
                        c  & d
                    \end{vmatrix}
                    \)
              \item \(
                    \begin{vmatrix}
                        ta & tb \\
                        c  & d
                    \end{vmatrix}
                    = t
                    \begin{vmatrix}
                        a & b \\
                        c & d
                    \end{vmatrix}
                    \)
          \end{enumerate}
    \item If \(\symbf{A}\) has two equal rows, then \(\det{\left(
          \symbf{A} \right)}=0\).
    \item Adding a scalar multiple of one row to another does not
          change the determinant of a matrix.
    \item If \(\symbf{A}\) has a row of zeros, then \(\det{\left(
          \symbf{A} \right)}=0\).
    \item If \(\symbf{A}\) is triangular, then \(\det{\left( \symbf{A}
          \right)}=\prod_{i=1}^{n} a_{ii}\).
    \item If \(\symbf{A}\) is singular, then \(\det{\left( \symbf{A}
          \right)}=0\).
    \item \(\det{\left( \symbf{A}\symbf{B} \right)}=\det{\left( \symbf{A} \right)}\det{\left( \symbf{B} \right)}\).
    \item \(\det{\left( \symbf{A}^\top \right)}=\det{\left( \symbf{A} \right)}\).
\end{enumerate}
\subsection{Matrix Minors}
\begin{definition}
    The \textbf{minor} of \(a_{ij}\) in \(\symbf{A}\), denoted
    \(\symbf{M}_{ij}\), is the determinant of the sub-matrix formed by deleting
    the \(i\)th row and \(j\)th column of \(\symbf{A}\).
\end{definition}
\subsection{Matrix Cofactors}
\begin{definition}
    The \textbf{cofactor} of \(a_{ij}\) in \(\symbf{A}\) is defined as
    \begin{equation*}
        \symbf{C}_{ij} = \left( -1 \right)^{i+j} \symbf{M}_{ij}
    \end{equation*}
\end{definition}
\subsection{The Determinant of a Matrix}
\begin{theorem}
    The determinant of an \(n\times n\) matrix \(\symbf{A}\) is given
    by
    \begin{equation*}
        \det{\left( \symbf{A} \right)} = \sum_{j=1}^n a_{ij}\symbf{C}_{ij} = \sum_{i=1}^n a_{ij}\symbf{C}_{ij}
    \end{equation*}
    where \(a_{ij}\) is the entry in the \(i\)th row and \(j\)th column
    of \(\symbf{A}\).
\end{theorem}
\subsection{The Cofactor Matrix}
\begin{definition}
    The \textbf{cofactor matrix} of an \(n\times n\) matrix
    \(\symbf{A}\), denoted \(\symbf{C}\), is defined as the matrix
    of the cofactors of \(\symbf{A}\).
    \begin{equation*}
        \symbf{C} =
        \begin{bmatrix}
            c_{11} & c_{12} & \cdots & c_{1n} \\
            c_{21} & c_{22} & \cdots & c_{2n} \\
            \vdots & \vdots &        & \vdots \\
            c_{n1} & c_{n2} & \cdots & c_{nn}
        \end{bmatrix}
    \end{equation*}
\end{definition}
\subsection{The Adjugate of a Matrix}
\begin{definition}
    The \textbf{adjugate} (or \textit{classical adjoint}) of a square
    matrix \(\symbf{A}\), denoted
    \(\adj{\left( \symbf{A} \right)}\), is the transpose of its cofactor
    matrix.
    \begin{equation*}
        \adj{\left( \symbf{A} \right)} = \symbf{C}^\top
    \end{equation*}
\end{definition}
\subsection{The Inverse of a Matrix}
\begin{theorem}
    The \textbf{inverse} of a non-singular matrix \(\symbf{A}\) is
    given by
    \begin{equation*}
        \symbf{A}^{-1}=\frac{1}{\det{\left( \symbf{A} \right)}} \adj{\left( \symbf{A} \right)}
    \end{equation*}
\end{theorem}
\section{Invariant Subspaces}
\begin{definition}
    Consider the subspace \(\mathcal{V}\) of the linear mapping
    \(T:V\rightarrow V\) from a vector space \(V\) to itself, then
    \(\mathcal{V}\) is an \textbf{invariant subspace} of \(T\) if
    \begin{equation*}
        T\left(\mathcal{V}\right)\subseteq \mathcal{V}
    \end{equation*}
\end{definition}
\begin{theorem}
    If \(\mathcal{V}\) is an invariant subspace of a linear mapping
    \(T: V \rightarrow V\) from a vector space \(V\) to itself, then
    \begin{equation*}
        \forall \symbf{v}\in \mathcal{V}\implies T\left(\symbf{v}\right)\in \mathcal{V}
    \end{equation*}
\end{theorem}
\subsection{Trivial Invariant Subspaces}
\begin{enumerate}
    \item \(V\).
    \item \(\left\{ \symbf{0} \right\}\).
    \item \(\vker{\left( T \right)}\).
    \item \(\vim{\left( T \right)}\).
    \item Any linear combination of invariant subspaces.
\end{enumerate}
\subsection{Eigenspaces}
\begin{definition}
    If an invariant subspace is one-dimensional, then the subspace is
    called an \textbf{eigenspace} of the linear transformation.
\end{definition}
\begin{theorem}
    If \(\mathcal{V}\) is an eigenspace of the linear mapping
    \(T: V \rightarrow V\), then
    \begin{equation*}
        \mathcal{V} = \left\{ \forall \symbf{v}\in \mathcal{V}:\exists \lambda \in \mathbb{C}:T\left(\symbf{v}\right)=\lambda \symbf{v} \right\}
    \end{equation*}
    where \(\lambda\) is the \textbf{eigenvalue} associated with the
    \textbf{eigenvector} \(\symbf{v}\).
\end{theorem}
\subsection{The Eigenvalue Problem}
\begin{theorem}
    The eigenvalues \(\lambda\) of an invertible square matrix
    \(\symbf{A}\), are the solutions to
    \begin{equation*}
        \det{\left( \symbf{A} - \lambda \symbf{I} \right)}=\symbf{0}
    \end{equation*}
\end{theorem}
\begin{theorem}
    The eigenvectors associated with each eigenvalue, of an invertible
    square matrix \(\symbf{A}\), are the solutions to
    \begin{equation*}
        \left( \symbf{A} - \lambda \symbf{I} \right) \symbf{v}=\symbf{0}
    \end{equation*}
\end{theorem}
\begin{proof}
    The eigenvalues and associated eigenvectors of a square matrix
    \(\symbf{A}\), are the solutions to
    \(\symbf{A}\symbf{v}=\lambda \symbf{v}\).
    \begin{align*}
        \symbf{A}\symbf{v}                                     & =\lambda \symbf{v} \\
        \symbf{A}\symbf{v} - \lambda \symbf{v}                 & =\symbf{0}         \\
        \left( \symbf{A} - \lambda \symbf{I} \right) \symbf{v} & =\symbf{0}
    \end{align*}
    The linear system
    \(\left( \symbf{A} - \lambda\symbf{I} \right) \symbf{v}=\symbf{0}\)
    has a nontrivial solution iff \(\symbf{A} - \lambda \symbf{I}\)
    is singular.
\end{proof}
\subsection{Properties of Eigenvalues}
\begin{theorem}
    \begin{equation*}
        \Tr{\left( \symbf{A} \right)} = \sum_{i=1}^n \lambda_i
    \end{equation*}
\end{theorem}
\begin{theorem}
    \begin{equation*}
        \det{\left( \symbf{A} \right)} = \prod_{i=1}^n \lambda_i
    \end{equation*}
\end{theorem}
\section{Eigen Decomposition}
\subsection{Similarity Transformations}
\begin{definition}
    A \textbf{similarity transformation} is a linear mapping of the form
    \begin{equation*}
        \symbf{A}\rightarrow \symbf{V}^{-1}\symbf{A}\symbf{V}
    \end{equation*}
    in which the matrices \(\symbf{A}\) and \(\symbf{V}\) are
    \(n \times n\) invertible matrices. Here we say, ``\(\symbf{A}\)
    is similar to \(\symbf{V}^{-1}\symbf{A}\symbf{V}\)''.
\end{definition}
\subsection{Matrix Diagonalisation}
\begin{definition}
    The matrix \(\symbf{A}\) is a \textbf{diagonalisable} matrix if it
    is similar to a diagonal matrix. That is, there exists an invertible
    matrix \(\symbf{V}\), and diagonal matrix \(\symbf{D}\),
    such that
    \begin{equation*}
        \symbf{D}=\symbf{V}^{-1}\symbf{A}\symbf{V}
    \end{equation*}
\end{definition}
\begin{theorem}
    Let \(\symbf{A}\) be an \(n \times n\) matrix with \(n\) linearly
    independent eigenvectors, then \(\symbf{A}\) is diagonalisable if
    \(\symbf{D}=\diag{\left( \lambda_1,\: \lambda_2,\: \dots,\: \lambda_n \right)}\)
    and \(\symbf{V}\) is a matrix composed of the eigenvectors of
    \(\symbf{A}\). Explicitly,
    \begin{equation*}
        \symbf{D} =
        \begin{bmatrix}
            \lambda_1 &           &        &           \\
                      & \lambda_2 &        &           \\
                      &           & \ddots &           \\
                      &           &        & \lambda_n
        \end{bmatrix}
        \quad
        \text{and}\quad
        \symbf{V} =
        \begin{bmatrix}
            \vertbar    & \vertbar    &        & \vertbar    \\
            \symbf{v}_1 & \symbf{v}_2 & \cdots & \symbf{v}_n \\
            \vertbar    & \vertbar    &        & \vertbar
        \end{bmatrix}
    \end{equation*}
    where \(\symbf{v}_1,\: \symbf{v}_2,\: \dots,\: \symbf{v}_n\)
    are the eigenvectors of \(\symbf{A}\).
\end{theorem}
\begin{proof}
    Let \(\symbf{v}_1,\: \symbf{v}_2,\: \dots,\: \symbf{v}_n\) be
    linearly independent eigenvectors of \(\symbf{A}\), and
    \(\lambda_1,\: \lambda_2,\: \dots,\: \lambda_n\), the associated
    eigenvalues. By definition of an eigenspace, we have
    \begin{equation*}
        \left\{
        \setlength\arraycolsep{0pt}
        \begin{array}
            { r >{{}}c<{{}} l }
            \symbf{A}\symbf{v}_1      & = & \lambda_1 \symbf{v}_1                            \\
            \symbf{A}\symbf{v}_2      & = & \lambda_2 \symbf{v}_2                            \\
            \vdotswithin{\symbf{v}_3} &   & \vdotswithin{\lambda_3}\vdotswithin{\symbf{v}_3} \\
            \symbf{A}\symbf{v}_n      & = & \lambda_n \symbf{v}_n
        \end{array}
        \right.
    \end{equation*}
    which we can rewrite as
    \begin{align}
        \symbf{A}\symbf{V} & = \symbf{V}\symbf{D} \label{equation:eigenspace_matrix} \\
        \symbf{D}          & =\symbf{V}^{-1}\symbf{A}\symbf{V} \nonumber
    \end{align}
    by rearranging
    \hyperref[equation:eigenspace_matrix]{Equation~\ref{equation:eigenspace_matrix}},
    we have \(\symbf{A}\) in terms of its eigenvalues and
    eigenvectors.
    \begin{equation*}
        \symbf{A} = \symbf{V}\symbf{D} \symbf{V}^{-1}
    \end{equation*}
\end{proof}
\subsection{Powers of a Matrix}
\begin{theorem}
    Let \(\symbf{A}\) be a diagonalisable matrix, then for all
    \(k \in \N_0\)
    \begin{equation*}
        \symbf{A}^k = \symbf{V} \symbf{D}^k \symbf{V}^{-1}
    \end{equation*}
\end{theorem}
\begin{proof}
    \begin{align*}
        \symbf{A}^k & = \left( \symbf{V} \symbf{D} \symbf{V}^{-1} \right)^k                                                                                                                                         \\
                    & = \underbrace{\left( \symbf{V} \symbf{D} \symbf{V}^{-1} \right)\left( \symbf{V} \symbf{D} \symbf{V}^{-1} \right)\cdots\left( \symbf{V} \symbf{D} \symbf{V}^{-1} \right)}_{\text{\(k\) times}} \\
                    & = \underbrace{\symbf{V} \symbf{D} \symbf{V}^{-1} \symbf{V} \symbf{D} \symbf{V}^{-1} \cdots \symbf{V} \symbf{D} \symbf{V}^{-1} }_{\text{\(k\) times}}                                          \\
                    & = \underbrace{\symbf{V} \symbf{D} \cancel{\symbf{V}^{-1} \symbf{V}} \symbf{D} \cancel{\symbf{V}^{-1} \cdots \symbf{V}} \symbf{D} \symbf{V}^{-1} }_{\text{\(k\) times}}                        \\
                    & = \symbf{V} \underbrace{ \symbf{D} \symbf{D} \cdots \symbf{D} }_{\text{\(k\) times}}\symbf{V}^{-1}                                                                                            \\
                    & = \symbf{V} \symbf{D}^k\symbf{V}^{-1}
    \end{align*}
\end{proof}
\begin{theorem}
    The eigenvalues of \(\symbf{A}^k\), \(\forall k \in \N\)
    are \(\lambda_1^k,\: \lambda_2^k,\: \dots,\: \lambda_n^k\).
\end{theorem}
\begin{theorem}
    The eigenvectors of \(\symbf{A}\) are equal to the eigenvectors of
    \(\symbf{A}^k\).
\end{theorem}
\section{System of Differential Equations}
\subsection{First-Order Differential Equations}
\begin{definition}
    A \textbf{first-order differential equation} is a differential
    equation where the highest derivative is of order one.
    \begin{equation*}
        x' = a x
    \end{equation*}
\end{definition}
\begin{theorem}
    The general solution to a first-order linear differential equation
    is of the form
    \begin{equation*}
        x\left( t \right) = c_1 e^{a t}
    \end{equation*}
    where \(c_1\) is an arbitrary constant.
\end{theorem}
\subsection{First-Order System of Differential Equations}
\begin{definition}
    A \textbf{first-order system of differential equations} is of the
    form
    \begin{equation*}
        \begin{cases}
            \begin{array}
                { c >{{}}c<{{}} c >{{}}c<{{}} c >{{}}c<{{}} c >{{}}c<{{}} c  }
                x'_1               & = & a_{11}x_1                         & + & a_{12}x_2                         & + & \cdots & + & a_{1n}x_n                         \\
                x'_2               & = & a_{21}x_1                         & + & a_{22}x_2                         & + & \cdots & + & a_{2n}x_n                         \\
                \vdotswithin{x'_3} &   & \vdotswithin{a_{31}}\phantom{x_1} &   & \vdotswithin{a_{32}}\phantom{x_2} &   &        &   & \vdotswithin{a_{3n}}\phantom{x_n} \\
                x'_n               & = & a_{n1}x_1                         & + & a_{n2}x_2                         & + & \cdots & + & a_{nn}x_n
            \end{array}
        \end{cases}
    \end{equation*}
    where \(x_1=x_1\left( t \right),\: x_2=x_2\left( t \right),\: \dots,\: x_n=x_n\left( t \right)\) are the
    functions to be determined. In matrix form, the system can be
    written as
    \begin{align*}
        \odv{}{t}
        \begin{bmatrix}
            x_1    \\
            x_2    \\
            \vdots \\
            x_n
        \end{bmatrix}
                   & =
        \begin{bmatrix}
            a_{11} & a_{12} & \cdots & a_{1n} \\
            a_{21} & a_{22} & \cdots & a_{2n} \\
            \vdots & \vdots &        & \vdots \\
            a_{n1} & a_{n2} & \cdots & a_{nn}
        \end{bmatrix}
        \begin{bmatrix}
            x_1    \\
            x_2    \\
            \vdots \\
            x_n
        \end{bmatrix}
        \\
        \symbf{x}' & = \symbf{A} \symbf{x}
    \end{align*}
\end{definition}
\subsection{Solution using Diagonalisation}
\begin{theorem}
    The first-order system of differential equations
    \(\symbf{x}' = \symbf{A} \symbf{x}\) can be solved using the
    following substitution
    \begin{equation*}
        \symbf{x}=\symbf{V}\symbf{u}
    \end{equation*}
    where \(\symbf{u}\) is a vector to be determined, and
    \(\symbf{V}\) is the matrix that diagonalises \(\symbf{A}\).
    \(\symbf{u}\) is determined by solving
    \begin{equation*}
        \symbf{u}' = \symbf{D} \symbf{u}
    \end{equation*}
    where \(\symbf{D}\) is the diagonal similarity
    transformation of \(\symbf{A}\). This substitution uncouples the
    system of differential equations so that each equation can be solved
    as a first-order differential equation.
\end{theorem}
\begin{proof}
    \begingroup
    \allowdisplaybreaks
    \begin{align*}
        \symbf{x}'                         & = \symbf{A} \symbf{x}                                                               \\
        \left( \symbf{V}\symbf{u} \right)' & = \left( \symbf{V}\symbf{D} \symbf{V}^{-1} \right)\left( \symbf{V}\symbf{u} \right) \\
        \symbf{V}\symbf{u}'                & = \symbf{V}\symbf{D} \symbf{V}^{-1} \symbf{V}\symbf{u}                              \\
        \cancel{\symbf{V}}\symbf{u}'       & = \cancel{\symbf{V}}\symbf{D} \cancel{\symbf{V}^{-1} \symbf{V}}\symbf{u}            \\
        \symbf{u}'                         & = \symbf{D} \symbf{u}
    \end{align*}
    \endgroup
\end{proof}
\begin{theorem}
    If \(\symbf{A}\) is a diagonalisable matrix, then the general
    solution of \(\symbf{x}' = \symbf{A} \symbf{x}\) can be
    expressed as
    \begin{equation*}
        \symbf{x}\left( t \right) = c_1 e^{\lambda_1 t} \symbf{v}_1 + c_2 e^{\lambda_2 t} \symbf{v}_2 + \cdots + c_n e^{\lambda_n t} \symbf{v}_n
    \end{equation*}
\end{theorem}
\subsection{Principle of Superposition}
\begin{theorem}
    If \(x_1\) and \(x_2\) are two solutions to a linear differential
    equation, then
    \begin{equation*}
        x = c_1 x_1 + c_2 x_2
    \end{equation*}
    is also a solution to the differential equation.
\end{theorem}
\subsection{Higher-Order Differential Equations}
\begin{theorem}
    A \textbf{higher-order linear differential equation} can be solved
    by first converting it to a first-order linear system. Consider the
    \(n\)th-order differential equation
    \begin{equation*}
        x^{\left( n \right)} + a_1 x^{\left( n-1 \right)} + \cdots + a_{n-1} x' + a_n x = 0
    \end{equation*}
    We then define
    \begin{align*}
        x_1 & = x                      \\
        x_2 & = x'                     \\
            & \vdotswithin{=}          \\
        x_n & = x^{\left( n-1 \right)}
    \end{align*}
    Let \(\symbf{x}=
    \begin{bmatrix}
        x_1 & x_2 & \cdots & x_n
    \end{bmatrix}
    ^\top
    \). Then the first-order linear system of differential equations can
    be written as
    \begin{equation*}
        \odv{}{t}
        \begin{bmatrix}
            x_1               \\
            x_2               \\
            \vdotswithin{x_3} \\
            x_n
        \end{bmatrix}
        =
        \begin{bmatrix}
            0      & 1        & 0        & \cdots & 0      \\
            0      & 0        & 1        & \cdots & 0      \\
            \vdots & \vdots   & \vdots   & \ddots & \vdots \\
            0      & 0        & 0        & \cdots & 1      \\
            -a_n   & -a_{n-1} & -a_{n-2} & \cdots & -a_1
        \end{bmatrix}
        \begin{bmatrix}
            x_1               \\
            x_2               \\
            \vdotswithin{x_3} \\
            x_n
        \end{bmatrix}
    \end{equation*}
\end{theorem}
\end{document}