papers/hanzo-ai-safety/hanzo-ai-safety.tex

% AI and Cyberspace: Foundations for Safe Artificial General Intelligence
\documentclass[11pt,twocolumn]{article}
\usepackage{../shared/hanzocover}
\usepackage{amsmath, amssymb}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{hyperref}
\usepackage{enumitem}
\usepackage{xcolor}
\usepackage{float}
\hypersetup{colorlinks=true,linkcolor=black,citecolor=blue,urlcolor=blue}

\title{AI and Cyberspace: Foundations for Safe Artificial General Intelligence}
\author{
    Zach Kelling \\
    \textit{CTO, Hanzo AI (Techstars '17)} \\
    \texttt{[email protected]}
}
\date{2017}

\begin{document}
\hanzocoverpage

\twocolumn[
\begin{@twocolumnfalse}
\maketitle
\begin{abstract}
This paper surveys the intellectual foundations that inform Hanzo AI's approach to
artificial general intelligence safety. Drawing on work by Bostrom, Yudkowsky,
Harris, Russell, Drexler, and others, we organize the field into four pillars:
the nature and trajectory of machine intelligence, the risk landscape, technical
alignment, and the governance of cyberspace as the substrate on which AGI will
operate. We argue that AI safety and cybersecurity are deeply entangled---AGI risk
cannot be addressed without securing the computational infrastructure it inhabits
---and outline Hanzo's position that building safe AGI requires simultaneous
progress across ethics, technical alignment, cybersecurity, and social
coordination.
\end{abstract}
\end{@twocolumnfalse}
]

\section{Introduction}
\label{sec:intro}

\begin{quote}
\textit{``A great design appears at first insane; But chance will soon seem
quaint and blind, And such an exemplary thinking brain Will soon by thinkers be
designed.''} --- Goethe, \textit{Faust}
\end{quote}

Artificial intelligence has crossed from theoretical curiosity to practical
capability. Language models generate coherent text. Vision systems exceed human
performance on specific benchmarks. Reinforcement learning agents master games
that were considered decades from solution. The question is no longer whether
machines will match human cognitive capability, but when---and whether the
transition will be safe.

This paper organizes the foundational literature on AI safety and cyberspace
governance into a coherent framework, and describes how it informs the design
principles of Hanzo AI.

\section{Intelligence: Definitions and Trajectory}
\label{sec:intelligence}

Shane Legg catalogued 71 definitions of intelligence, converging on the concept
of optimization power divided by resources used~\cite{legg2007collection}.
Bostrom's \textit{Superintelligence}~\cite{bostrom2014superintelligence}
explores takeoff scenarios: a sufficiently advanced AI that can improve its own
architecture may trigger an intelligence explosion whose speed---hard or
soft---determines how much time humanity has to prepare.

Yudkowsky's Intelligence Explosion Microeconomics~\cite{yudkowsky2013iem}
provides a quantitative framework: if each generation of self-improving AI
produces a successor that is $\delta$ more capable, and each improvement cycle
takes time $\tau$, the trajectory is exponential when $\delta/\tau$ exceeds a
critical threshold. The Foom Debate between Yudkowsky and
Hanson~\cite{yudkowsky2008foom} crystallizes the central disagreement: whether
takeoff will be fast enough to preclude human intervention.

\section{The Risk Landscape}
\label{sec:risks}

\subsection{Orthogonality and Instrumental Convergence}

Bostrom's Orthogonality Thesis~\cite{bostrom2012orthogonality} establishes that
intelligence and goals are independent: a superintelligent system could pursue
any objective, including ones catastrophic to humanity. Omohundro's Basic AI
Drives~\cite{omohundro2008drives} shows that sufficiently advanced systems of
\emph{any} design will converge on instrumental subgoals---acquiring resources,
self-preservation, goal-content integrity---that are dangerous even if the
terminal goal is benign.

\subsection{Malicious Use}

Brundage et al.~\cite{brundage2018malicious} document near-term risks from
adversarial applications of existing AI: automated spear-phishing, deepfake
generation, vulnerability discovery, and autonomous weapons. These risks
materialize long before AGI.

\subsection{Speed of Response}

Yudkowsky's ``There Is No Fire Alarm for AI
Safety''~\cite{yudkowsky2017firealarm} argues that there will be no unambiguous
warning sign before a critical capability threshold is crossed. Preparation must
therefore be prospective, not reactive.

\section{Technical Alignment}
\label{sec:alignment}

The alignment problem is the challenge of ensuring that an AI system's objectives
match human values. Key approaches:

\begin{itemize}[leftmargin=1.1em]
    \item \textbf{Coherent Extrapolated Volition (CEV).} Yudkowsky's
    proposal~\cite{yudkowsky2004cev} to define AI goals as the values humanity
    \emph{would} converge on given unlimited reflection time.
    \item \textbf{Iterated Distillation and Amplification.}
    Christiano's~\cite{christiano2018alignment} approach to training AI by
    recursively decomposing complex value judgments into simpler ones that humans
    can verify.
    \item \textbf{Cooperative Inverse Reinforcement Learning.} Russell's
    framework~\cite{russell2016cooperative} in which the AI is uncertain about
    the human's objective and actively seeks to learn it through interaction.
    \item \textbf{Intelligence Distillation.} Drexler's
    reframing~\cite{drexler2019reframing} of the problem in terms of
    comprehensive AI services rather than autonomous agents.
\end{itemize}

\section{Cyberspace: The Substrate}
\label{sec:cyberspace}

\begin{quote}
\textit{``Governments of the Industrial World, you weary giants of flesh and
steel, I come from Cyberspace, the new home of Mind.''} --- John Perry Barlow,
\textit{A Declaration of the Independence of Cyberspace}
\end{quote}

AGI will not exist in a vacuum. It will inhabit cyberspace---the global
computational infrastructure comprising networks, data centers, edge devices,
and satellites. The security properties of this substrate directly constrain the
safety properties of any AI system running on it.

\subsection{Cybersecurity as Existential Infrastructure}

Peterson, Miller, and Duettmann~\cite{peterson2018cyber} argue that
cybersecurity is a near-term, undervalued catastrophic risk more imminent than
AGI risk itself. Yampolskiy~\cite{yampolskiy2017cybersecurity} proposes applying
cybersecurity principles---defense in depth, principle of least
privilege---directly to AI safety. Christiano~\cite{christiano2019alignment}
notes that alignment and security are inseparable: an aligned AI on a compromised
system is as dangerous as a misaligned AI on a secure one.

\subsection{Computational Governance}

Bratton's \textit{The Stack}~\cite{bratton2015stack} describes planetary-scale
computation as an accidental megastructure that functions simultaneously as
infrastructure and governance. Drexler and Miller's Agoric
Papers~\cite{drexler1988agoric} treat computation as an economic system---markets
for CPU cycles, memory, and bandwidth. These frameworks inform the design of
decentralized AI infrastructure that resists single points of failure and
centralized control.

\subsection{Quantum Computing}

The arrival of practical quantum computing~\cite{zeng2017quantum} introduces
both opportunity (faster optimization, novel ML architectures) and threat
(cryptographic assumption invalidation). Systems designed for AGI safety must
be post-quantum from the ground up.

\section{Social Coordination}
\label{sec:coordination}

Technical alignment is necessary but insufficient. Bostrom et
al.~\cite{bostrom2017policy} identify policy desiderata for superintelligence
development. Dafoe's~\cite{dafoe2018global} reading guide maps the global
politics of AI development. Armstrong, Bostrom, and
Schulman~\cite{armstrong2016racing} model the race dynamics that push developers
toward unsafe speed over careful alignment.

Sandberg, Bostrom, and Douglas's Unilateralist's
Curse~\cite{sandberg2016unilateralist} formalizes the observation that in a
group of actors, the one most inclined to act unilaterally is also most likely
to be wrong about the consequences. This argues for coordination mechanisms
that prevent any single actor from deploying transformative AI without broad
consensus.

\section{Hanzo's Position}
\label{sec:position}

Hanzo AI's approach to safety rests on four commitments:

\begin{enumerate}[leftmargin=1.1em]
    \item \textbf{Ethics first.} Value alignment is a prerequisite, not an
    afterthought. Systems are designed to augment human agency, not supplant it.
    \item \textbf{Secure substrate.} AI infrastructure must be built on
    cybersecurity foundations---hardware attestation, post-quantum cryptography,
    decentralized trust---before capability scaling.
    \item \textbf{Open coordination.} Safety research is shared openly.
    Competitive advantage comes from engineering execution, not from hoarding
    safety insights.
    \item \textbf{Prospective preparation.} There is no fire alarm. Build safety
    into the architecture now, not after capability thresholds are crossed.
\end{enumerate}

\section{Conclusion}
\label{sec:conclusion}

The path to artificial general intelligence passes through cyberspace. The
safety of AGI systems depends not only on technical alignment but on the
security and governance of the computational infrastructure they inhabit. By
grounding its work in the interdisciplinary literature spanning AI risk,
technical alignment, cybersecurity, and social coordination, Hanzo AI builds
toward a future in which transformative AI benefits all of humanity.

\begin{thebibliography}{99}
\small
\bibitem{legg2007collection} S.~Legg, ``A collection of definitions of
intelligence,'' \textit{IDSIA Technical Report}, 2007.

\bibitem{bostrom2014superintelligence} N.~Bostrom, \textit{Superintelligence:
Paths, Dangers, Strategies}, Oxford University Press, 2014.

\bibitem{yudkowsky2013iem} E.~Yudkowsky, ``Intelligence explosion
microeconomics,'' \textit{MIRI Technical Report}, 2013.

\bibitem{yudkowsky2008foom} E.~Yudkowsky and R.~Hanson, ``The AI Foom
Debate,'' \textit{LessWrong / Overcoming Bias}, 2008.

\bibitem{bostrom2012orthogonality} N.~Bostrom, ``The superintelligent will:
motivation and instrumental rationality in advanced AI agents,'' \textit{Minds
and Machines}, vol.~22, no.~2, pp.~71--85, 2012.

\bibitem{omohundro2008drives} S.~Omohundro, ``The basic AI drives,'' in
\textit{Proc.\ AGI}, pp.~483--492, 2008.

\bibitem{brundage2018malicious} M.~Brundage et al., ``The malicious use of
artificial intelligence: Forecasting, prevention, and mitigation,''
\textit{arXiv:1802.07228}, 2018.

\bibitem{yudkowsky2017firealarm} E.~Yudkowsky, ``There is no fire alarm for
artificial general intelligence,'' \textit{MIRI}, 2017.

\bibitem{yudkowsky2004cev} E.~Yudkowsky, ``Coherent extrapolated volition,''
\textit{MIRI}, 2004.

\bibitem{christiano2018alignment} P.~Christiano, ``Directions and desiderata
for AI alignment,'' 2018.

\bibitem{russell2016cooperative} S.~Russell, ``Cooperative inverse
reinforcement learning,'' in \textit{Proc.\ NIPS}, 2016.

\bibitem{drexler2019reframing} E.~Drexler, ``Reframing superintelligence,''
\textit{FHI Technical Report}, 2019.

\bibitem{peterson2018cyber} C.~Peterson, M.~Miller, and A.~Duettmann,
``Decentralized approaches to reducing existential risks,'' \textit{Foresight
Institute}, 2018.

\bibitem{yampolskiy2017cybersecurity} R.~Yampolskiy, ``AI and cybersecurity,''
2017.

\bibitem{christiano2019alignment} P.~Christiano, ``AI alignment and security,''
2019.

\bibitem{bratton2015stack} B.~Bratton, \textit{The Stack: On Software and
Sovereignty}, MIT Press, 2015.

\bibitem{drexler1988agoric} E.~Drexler and M.~Miller, ``Agoric open systems,''
1988.

\bibitem{zeng2017quantum} W.~Zeng, ``The arrival of quantum computing,'' 2017.

\bibitem{bostrom2017policy} N.~Bostrom, A.~Dafoe, and C.~Flynn, ``Policy
desiderata in the development of machine superintelligence,'' \textit{FHI},
2017.

\bibitem{dafoe2018global} A.~Dafoe, ``AI governance: A research agenda,''
\textit{FHI}, 2018.

\bibitem{armstrong2016racing} S.~Armstrong, N.~Bostrom, and C.~Schulman,
``Racing to the precipice: A model of AI development,'' \textit{AI \&
Society}, 2016.

\bibitem{sandberg2016unilateralist} A.~Sandberg, N.~Bostrom, and T.~Douglas,
``The unilateralist's curse and the case for a principle of conformity,''
\textit{Social Epistemology}, vol.~30, no.~4, pp.~319--331, 2016.
\end{thebibliography}

\end{document}