\documentclass[12pt]{book}
\usepackage{boxedminipage}
\usepackage{kbordermatrix}
\input 312-fall2016.sty

\newcommand{\bhat}{\widehat b}
\newcommand{\chat}{\widehat c}
\newcommand{\dhat}{\widehat d}
\newcommand{\yhat}{\widehat y}
\newcommand{\that}{\widehat t}
\newcommand{\ybar}{\overline y}
\newcommand{\zbar}{\overline z}
\def\Rlang/{{\bfseries R}}


\begin{document}

\setcounter{chapter}{0}
\chapter{Least squares when $X$ is not of full rank}
\today

\TOC
\bigskip

\copyright David Pollard 2016

\section{Least squares}

Consider the least squares problem where the $n\times p$ matrix~$X$ of predictors
has rank~$m$, which is~$<p$. For simplicity, suppose the columns of~$X$ have been rearranged so that
$x_1,\dots,x_m$ are linearly independent, so that $x_j$ for $j>m$ could be written as a
 linear combination of the first~$m$. (The LINPACK 
 QR algorithm \cite[Chapter~9]{LINPACK} permutes the order of the columns as it applies the
 Householder transformations.)
 Partition~$X$ and the matrices~$Q$ accordingly
\beqN
X=\kbordermatrix{
~&m&p-m\cr
n&X_1&X_2
}
\AND/
Q=\kbordermatrix{
~&m&p-m&n-p\cr
n&Q_1&Q_2&Q_3
}.
\eeqN
(The little numbers above and to the left of the matrices show the sizes of the blocks.)
The QR algorithm arranges that the upper triangular matrix has a corresponding partition.
\bAlign
Q^T X &= 
\kbordermatrix{
~&n
\cr
m&Q_1^T
\cr
p-m&Q_2^T
\cr
n-p&Q_3^T
}
\bMatrix
X_1,X_2
\endbMatrix
=
\kbordermatrix{
~&m&p-m
\cr
m&Q_1^TX_1& Q_1^T X_2
\cr
p-m&Q_2^TX_1& Q_2^T X_2
\cr
n-p&Q_3^TX_1& Q_3^T X_2
}
\\
&=
\kbordermatrix{
~&p\cr
p&R\cr
n-p&0
}
=
\kbordermatrix{
~&m&p-m\cr
m&R_1&R_2
\cr
p-m&0&0\cr
n-p&0&0
}
\eAlign
so that
\beqN
\bMatrix
X_1&X_2
\endbMatrix
= Q
\bMatrix
R\cr
0
\endbMatrix
=
\bMatrix
Q_1 R_1&Q_1 R_2
\cr
0&0
\endbMatrix
\eeqN
That is, $X_1=Q_1R_1$ and $X_2=Q_1R_2$.

The $m\times m$ matrix~$R_1$ is upper triangular with nonzero elements down its diagonal; it has an inverse. The columns of~$Q_1$ provide an orthonormal basis for the subspace~$\xx$ spanned by the columns of~$X$, which is the same as the subspace
spanned by the columns of~$X_1$. The columns of~$Q_2$ and~$Q_3$ span the subspace~$\xx^\perp$. Thus
\beqN
Q_1^T X_1 = R_1
\AND/
Q_1^T
\eeqN
so that
\beqN
X_1 = Q_1R_1
\eeqN






\bibliographystyle{chicago}
\bibliography{/Users/pollard/Dropbox/texmf/bibtex/bib/DBP}

\end{document}