meineBibliothek.bib

@online{noauthor_how_2018,
	title = {How Do We Align Artificial Intelligence with Human Values? German},
	url = {https://futureoflife.org/2018/03/29/how-do-we-align-artificial-intelligence-with-human-values-german/},
	shorttitle = {How Do We Align Artificial Intelligence with Human Values?},
	titleaddon = {Future of Life Institute},
	urldate = {2019-07-08},
	date = {2018-03-29},
	langid = {american},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\SE9TM3C3\\how-do-we-align-artificial-intelligence-with-human-values-german.html:text/html}
}

@inreference{noauthor_machine_2019,
	title = {Machine Intelligence Research Institute},
	rights = {Creative Commons Attribution-{ShareAlike} License},
	url = {https://en.wikipedia.org/w/index.php?title=Machine_Intelligence_Research_Institute&oldid=897482914},
	abstract = {The Machine Intelligence Research Institute ({MIRI}), formerly the Singularity Institute for Artificial Intelligence ({SIAI}), is a non-profit organization founded in 2000 by Eliezer Yudkowsky, originally to accelerate the development of artificial intelligence, but focused since 2005 on identifying and managing the potential risks to humanity that future {AI} systems could become superintelligent.  {MIRI}'s work has focused on a friendly {AI} approach to system design and on predicting the rate of technology development.},
	booktitle = {Wikipedia},
	urldate = {2019-07-09},
	date = {2019-05-17},
	langid = {english},
	note = {Page Version {ID}: 897482914},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\BIEIMQJV\\index.html:text/html}
}

@inreference{noauthor_existential_2019,
	title = {Existential risk from artificial general intelligence},
	rights = {Creative Commons Attribution-{ShareAlike} License},
	url = {https://en.wikipedia.org/w/index.php?title=Existential_risk_from_artificial_general_intelligence&oldid=903450901},
	abstract = {Existential risk from artificial general intelligence is the hypothesis that substantial progress in artificial general intelligence ({AGI}) could someday result in human extinction or some other unrecoverable global catastrophe. For instance, the human species currently dominates other species because the human brain has some distinctive capabilities that other animals lack. If {AI} surpasses humanity in general intelligence and becomes "superintelligent", then this new superintelligence could become powerful and difficult to control. Just as the fate of the mountain gorilla depends on human goodwill, so might the fate of humanity depend on the actions of a future machine superintelligence.The likelihood of this type of scenario is widely debated, and hinges in part on differing scenarios for future progress in computer science. Once the exclusive domain of science fiction, concerns about superintelligence started to become mainstream in the 2010s, and were popularized by public figures such as Stephen Hawking, Bill Gates, and Elon Musk.One source of concern is that a sudden and unexpected "intelligence explosion" might take an unprepared human race by surprise. For example, in one scenario, the first-generation computer program found able to broadly match the effectiveness of an {AI} researcher is able to rewrite its algorithms and double its speed or capabilities in six months of massively parallel processing time. The second-generation program is expected to take three months to perform a similar chunk of work, on average; in practice, doubling its own capabilities may take longer if it experiences a mini-"{AI} winter", or may be quicker if it undergoes a miniature "{AI} Spring" where ideas from the previous generation are especially easy to mutate into the next generation. In this scenario the system undergoes an unprecedently large number of generations of improvement in a short time interval, jumping from subhuman performance in many areas to superhuman performance in all relevant areas. More broadly, examples like arithmetic and Go show that progress from human-level {AI} to superhuman ability is sometimes extremely rapid.A second source of concern is that controlling a superintelligent machine (or even instilling it with human-compatible values) may be an even harder problem than naïvely supposed. Some {AGI} researchers believe that a superintelligence would naturally resist attempts to shut it off, and that preprogramming a superintelligence with complicated human values may be an extremely difficult technical task. In contrast, skeptics such as Facebook's Yann {LeCun} argue that superintelligent machines will have no desire for self-preservation.},
	booktitle = {Wikipedia},
	urldate = {2019-07-09},
	date = {2019-06-25},
	langid = {english},
	note = {Page Version {ID}: 903450901},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\JX3GPRG7\\index.html:text/html}
}

@inreference{noauthor_superintelligence_2019,
	title = {Superintelligence},
	rights = {Creative Commons Attribution-{ShareAlike} License},
	url = {https://en.wikipedia.org/w/index.php?title=Superintelligence&oldid=904476345},
	abstract = {A superintelligence is a hypothetical agent that possesses intelligence far surpassing that of the brightest and most gifted human minds. "Superintelligence" may also refer to a property of problem-solving systems (e.g., superintelligent language translators or engineering assistants) whether or not these high-level intellectual competencies are embodied in agents that act in the world. A superintelligence may or may not be created by an intelligence explosion and associated with a technological singularity.
University of Oxford philosopher Nick Bostrom defines superintelligence as "any intellect that greatly exceeds the cognitive performance of humans in virtually all domains of interest". The program Fritz falls short of superintelligence even though it is much better than humans at chess because Fritz cannot outperform humans in other tasks. Following Hutter and Legg, Bostrom treats superintelligence as general dominance at goal-oriented behavior, leaving open whether an artificial or human superintelligence would possess capacities such as intentionality (cf. the Chinese room argument) or first-person consciousness (cf. the hard problem of consciousness).
Technological researchers disagree about how likely present-day human intelligence is to be surpassed. Some argue that advances in artificial intelligence ({AI}) will probably result in general reasoning systems that lack human cognitive limitations. Others believe that humans will evolve or directly modify their biology so as to achieve radically greater intelligence. A number of futures studies scenarios combine elements from both of these possibilities, suggesting that humans are likely to interface with computers, or upload their minds to computers, in a way that enables substantial intelligence amplification.
Some researchers believe that superintelligence will likely follow shortly after the development of artificial general intelligence. The first generally intelligent machines are likely to immediately hold an enormous advantage in at least some forms of mental capability, including the capacity of perfect recall, a vastly superior knowledge base, and the ability to multitask in ways not possible to biological entities. This may give them the opportunity to—either as a single being or as a new species—become much more powerful than humans, and to displace them.A number of scientists and forecasters argue for prioritizing early research into the possible benefits and risks of human and machine cognitive enhancement, because of the potential social impact of such technologies.},
	booktitle = {Wikipedia},
	urldate = {2019-07-09},
	date = {2019-07-02},
	langid = {english},
	note = {Page Version {ID}: 904476345},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\5SR3UQTD\\index.html:text/html}
}

@inreference{noauthor_friendly_2019,
	title = {Friendly artificial intelligence},
	rights = {Creative Commons Attribution-{ShareAlike} License},
	url = {https://en.wikipedia.org/w/index.php?title=Friendly_artificial_intelligence&oldid=903968334},
	abstract = {A friendly artificial intelligence (also friendly {AI} or {FAI}) is a hypothetical artificial general intelligence ({AGI}) that would have a positive effect on humanity. It is a part of the ethics of artificial intelligence and is closely related to machine ethics. While machine ethics is concerned with how an artificially intelligent agent should behave, friendly artificial intelligence research is focused on how to practically bring about this behaviour and ensuring it is adequately constrained.},
	booktitle = {Wikipedia},
	urldate = {2019-07-09},
	date = {2019-06-29},
	langid = {english},
	note = {Page Version {ID}: 903968334},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\E8KPVCZG\\index.html:text/html}
}

@online{noauthor_artificial_nodate,
	title = {Artificial Intelligence @ {MIRI}},
	url = {https://intelligence.org/},
	abstract = {{MIRI}'s artificial intelligence research is focused on developing the mathematical theory of trustworthy reasoning for advanced autonomous {AI} systems.},
	titleaddon = {Machine Intelligence Research Institute},
	urldate = {2019-07-09},
	langid = {american},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\P2CT36PL\\intelligence.org.html:text/html}
}

@online{noauthor_ai_2014,
	title = {{AI} Impacts},
	url = {https://aiimpacts.org/},
	abstract = {The {AI} Impacts project aims to improve our understanding of the long term impacts of artificial intelligence.

This site is a collection of reference pages outlining our best understanding of many relevant considerations.

You can browse a selection of pages from here, use the search or sitemap to answer specific questions, or see our blog for highlights and discussion.},
	titleaddon = {{AI} Impacts},
	urldate = {2019-07-09},
	date = {2014-12-19},
	langid = {american}
}

@inreference{noauthor_ai_2019,
	title = {{AI} takeover},
	rights = {Creative Commons Attribution-{ShareAlike} License},
	url = {https://en.wikipedia.org/w/index.php?title=AI_takeover&oldid=903455994},
	abstract = {An {AI} takeover is a hypothetical scenario in which artificial intelligence ({AI}) becomes the dominant form of intelligence on Earth, with computers or robots effectively taking control of the planet away from the human species. Possible scenarios include replacement of the entire human workforce, takeover by a superintelligent {AI}, and the popular notion of a robot uprising. Some public figures, such as Stephen Hawking and Elon Musk, have advocated research into precautionary measures to ensure future superintelligent machines remain under human control. Robot rebellions have been a major theme throughout science fiction for many decades though the scenarios dealt with by science fiction are generally very different from those of concern to scientists.},
	booktitle = {Wikipedia},
	urldate = {2019-07-09},
	date = {2019-06-25},
	langid = {english},
	note = {Page Version {ID}: 903455994},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\LJHMPX7U\\index.html:text/html}
}

@inreference{noauthor_ai_2019-1,
	title = {{AI} control problem},
	rights = {Creative Commons Attribution-{ShareAlike} License},
	url = {https://en.wikipedia.org/w/index.php?title=AI_control_problem&oldid=901891171},
	abstract = {In artificial intelligence ({AI}) and philosophy, the {AI} control problem is the issue of how to build a superintelligent agent that will aid its creators, and avoid inadvertently building a superintelligence that will harm its creators. Its study is motivated by the claim that the human race will have to get the control problem right "the first time", as a misprogrammed superintelligence might rationally decide to "take over the world" and refuse to permit its programmers to modify it after launch. In addition, some scholars argue that solutions to the control problem, alongside other advances in "{AI} safety engineering", might also find applications in existing non-superintelligent {AI}. Potential strategies include "capability control" (preventing an {AI} from being able to pursue harmful plans), and "motivational control" (building an {AI} that wants to be helpful).},
	booktitle = {Wikipedia},
	urldate = {2019-07-09},
	date = {2019-06-15},
	langid = {english},
	note = {Page Version {ID}: 901891171},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\5MGD3ZA3\\index.html:text/html}
}

@inreference{noauthor_kunstliche_2019,
	title = {Künstliche Intelligenz},
	rights = {Creative Commons Attribution-{ShareAlike} License},
	url = {https://de.wikipedia.org/w/index.php?title=K%C3%BCnstliche_Intelligenz&oldid=189910972},
	abstract = {Künstliche Intelligenz ({KI}, auch Artifizielle Intelligenz ({AI} bzw. A. I.), englisch artificial intelligence, {AI}) ist ein Teilgebiet der Informatik, welches sich mit der Automatisierung intelligenten Verhaltens und dem Maschinellen Lernen befasst. Der Begriff ist insofern nicht eindeutig abgrenzbar, als es bereits an einer genauen Definition von „Intelligenz“ mangelt. Dennoch wird er in Forschung und Entwicklung verwendet.},
	booktitle = {Wikipedia},
	urldate = {2019-07-09},
	date = {2019-06-27},
	langid = {german},
	note = {Page Version {ID}: 189910972},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\2VMQ4EUF\\index.html:text/html}
}

@inreference{noauthor_artificial_2019,
	title = {Artificial intelligence},
	rights = {Creative Commons Attribution-{ShareAlike} License},
	url = {https://en.wikipedia.org/w/index.php?title=Artificial_intelligence&oldid=905405777},
	abstract = {In computer science,  artificial intelligence ({AI}), sometimes called machine intelligence, is intelligence demonstrated by machines, in contrast to the natural intelligence displayed by humans.  Colloquially, the term "artificial intelligence" is often used to describe machines (or computers) that mimic "cognitive" functions that humans associate with the human mind, such as "learning" and "problem solving".As machines become increasingly capable, tasks considered to require "intelligence" are often removed from the definition of {AI}, a phenomenon known as the {AI} effect. A quip in Tesler's Theorem says "{AI} is whatever hasn't been done yet." For instance, optical character recognition is frequently excluded from things considered to be {AI}, having become a routine technology. Modern machine capabilities generally classified as {AI} include successfully understanding human speech, competing at the highest level in strategic game systems (such as chess and Go), autonomously operating cars, intelligent routing in content delivery networks, and military simulations.
Artificial intelligence can be classified into three different types of systems: analytical, human-inspired, and humanized artificial intelligence. Analytical {AI} has only characteristics consistent with cognitive intelligence; generating cognitive representation of the world and using learning based on past experience to inform future decisions. Human-inspired {AI} has elements from cognitive and emotional intelligence; understanding human emotions, in addition to cognitive elements, and considering them in their decision making. Humanized {AI} shows characteristics of all types of competencies (i.e., cognitive, emotional, and social intelligence), is able to be self-conscious and is self-aware in interactions with others.
Artificial intelligence was founded as an academic discipline in 1956, and in the years since has experienced several waves of optimism, followed by disappointment and the loss of funding (known as an "{AI} winter"), followed by new approaches, success and renewed funding. For most of its history, {AI} research has been divided into subfields that often fail to communicate with each other. These sub-fields are based on technical considerations, such as particular goals (e.g. "robotics" or "machine learning"), the use of particular tools ("logic" or artificial neural networks), or deep philosophical differences. Subfields have also been based on social factors (particular institutions or the work of particular researchers).The traditional problems (or goals) of {AI} research include reasoning, knowledge representation, planning, learning, natural language processing, perception and the ability to move and manipulate objects. General intelligence is among the field's long-term goals. Approaches include statistical methods, computational intelligence, and traditional symbolic {AI}. Many tools are used in {AI}, including versions of search and mathematical optimization, artificial neural networks, and methods based on statistics, probability and economics. The {AI} field draws upon computer science, information engineering, mathematics, psychology, linguistics, philosophy, and many other fields.
The field was founded on the claim that human intelligence "can be so precisely described that a machine can be made to simulate it". This raises philosophical arguments about the nature of the mind and the ethics of creating artificial beings endowed with human-like intelligence which are issues that have been explored by myth, fiction and philosophy since antiquity. Some people also consider {AI} to be a danger to humanity if it progresses unabated. Others believe that {AI}, unlike previous technological revolutions, will create a risk of mass unemployment.In the twenty-first century, {AI} techniques have experienced a resurgence following concurrent advances in computer power, large amounts of data, and theoretical understanding; and {AI} techniques have become an essential part of the technology industry, helping to solve many challenging problems in computer science, software engineering and operations research.},
	booktitle = {Wikipedia},
	urldate = {2019-07-09},
	date = {2019-07-08},
	langid = {english},
	note = {Page Version {ID}: 905405777},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\P2ZAHVZN\\index.html:text/html}
}

@inproceedings{okal_learning_2016,
	location = {Stockholm, Sweden},
	title = {Learning socially normative robot navigation behaviors with Bayesian inverse reinforcement learning},
	isbn = {978-1-4673-8026-3},
	url = {http://ieeexplore.ieee.org/document/7487452/},
	doi = {10.1109/ICRA.2016.7487452},
	eventtitle = {2016 {IEEE} International Conference on Robotics and Automation ({ICRA})},
	pages = {2889--2895},
	booktitle = {2016 {IEEE} International Conference on Robotics and Automation ({ICRA})},
	publisher = {{IEEE}},
	author = {Okal, Billy and Arras, Kai O.},
	urldate = {2019-07-09},
	date = {2016-05},
	file = {Okal und Arras - 2016 - Learning socially normative robot navigation behav.pdf:C\:\\Users\\fsram\\Zotero\\storage\\QDGQSY99\\Okal und Arras - 2016 - Learning socially normative robot navigation behav.pdf:application/pdf}
}

@article{erber_typen_nodate,
	title = {Typen Ku¨nstlicher Neuronaler Netze ———– Darstellung, Vergleich und Untersuchung der Anwendungsm¨oglichkeiten},
	pages = {83},
	author = {Erber, Lars},
	langid = {german},
	file = {Erber - Typen Ku¨nstlicher Neuronaler Netze ———– Darstellu.pdf:C\:\\Users\\fsram\\Zotero\\storage\\IMLRKKZD\\Erber - Typen Ku¨nstlicher Neuronaler Netze ———– Darstellu.pdf:application/pdf}
}

@article{arnold_value_nodate,
	title = {Value Alignment or Misalignment -- What Will Keep Systems Accountable?},
	abstract = {Machine learning’s advances have led to new ideas about the feasibility and importance of machine ethics keeping pace, with increasing emphasis on safety, containment, and alignment. This paper addresses a recent suggestion that inverse reinforcement learning ({IRL}) could be a means to so-called “value alignment.” We critically consider how such an approach can engage the social, norm-infused nature of ethical action and outline several features of ethical appraisal that go beyond simple models of behavior, including unavoidably temporal dimensions of norms and counterfactuals. We propose that a hybrid approach for computational architectures still offers the most promising avenue for machines acting in an ethical fashion.},
	pages = {8},
	author = {Arnold, Thomas and Kasenberg, Daniel and Scheutz, Matthias},
	langid = {english},
	file = {Arnold et al. - Value Alignment or Misalignment -- What Will Keep .pdf:C\:\\Users\\fsram\\Zotero\\storage\\929ILADP\\Arnold et al. - Value Alignment or Misalignment -- What Will Keep .pdf:application/pdf}
}

@online{christiano_reward_2016,
	title = {The reward engineering problem - {AI} Alignment},
	url = {https://ai-alignment.com/the-reward-engineering-problem-30285c779450},
	abstract = {How can we define rewards which incentivize weak {RL} agents to behave in a desirable way?},
	titleaddon = {Medium},
	author = {Christiano, Paul},
	urldate = {2019-07-09},
	date = {2016-10-13},
	langid = {english}
}

@video{piero_scaruffi_stuart_nodate,
	title = {Stuart Russell  on "The long-term future of (Artificial) Intelligence"},
	url = {https://www.youtube.com/watch?v=mukaRhQTMP8&feature=youtu.be},
	author = {{Piero Scaruffi}},
	urldate = {2019-07-09}
}

@article{cindy_mason_engineering_2015,
	title = {Engineering Kindness: Building A Machine With Compassionate Intelligence},
	url = {https://www.academia.edu/15865212/Engineering_Kindness_Building_A_Machine_With_Compassionate_Intelligence},
	shorttitle = {Engineering Kindness},
	journaltitle = {International Journal of Synthetic Emotion},
	author = {Cindy Mason and Mason, Cindy},
	date = {2015},
	file = {Engineering_Kindness_Building_A_Machine.pdf:C\:\\Users\\fsram\\Zotero\\storage\\I7ZMIUUJ\\Engineering_Kindness_Building_A_Machine.pdf:application/pdf;Friendly artificial intelligence - Lesswrongwiki:C\:\\Users\\fsram\\Zotero\\storage\\57K59JGN\\Friendly_artificial_intelligence.html:text/html}
}

@inreference{noauthor_friendly_2019-1,
	title = {Friendly artificial intelligence},
	url = {https://wiki.lesswrong.com/wiki/Friendly_artificial_intelligence},
	abstract = {A Friendly Artificial Intelligence (Friendly {AI}, or {FAI}) is a superintelligence (i.e., a really powerful optimization process) that produces good, beneficial outcomes rather than harmful ones. The term was coined by Eliezer Yudkowsky, so it is frequently associated with Yudkowsky's proposals for how an artificial general intelligence ({AGI}) of this sort would behave.

"Friendly {AI}" can also be used as a shorthand for Friendly {AI} theory, the field of knowledge concerned with building such an {AI}. Note that "Friendly" (with a capital "F") is being used as a term of art, referring specifically to {AIs} that promote humane values. An {FAI} need not be "friendly" in the conventional sense of being personable, compassionate, or fun to hang out with. Indeed, an {FAI} need not even be sentient.},
	booktitle = {{LessWrong}},
	date = {2019-06-29},
	langid = {english}
}

@online{noauthor_kunstliche_nodate,
	title = {Künstliche Intelligenz: Ethikleitlinien},
	url = {https://ec.europa.eu/commission/news/artificial-intelligence-2019-apr-08_de},
	shorttitle = {Künstliche Intelligenz},
	abstract = {Kommission treibt Arbeit an Ethikleitlinien weiter voran},
	titleaddon = {{EU}-Kommission - European Commission},
	type = {Text},
	urldate = {2019-07-15},
	langid = {german},
	file = {EthicsguidelinesfortrustworthyAI-DEpdf.pdf:C\:\\Users\\fsram\\Zotero\\storage\\NT424EBY\\EthicsguidelinesfortrustworthyAI-DEpdf.pdf:application/pdf;Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\54LRYVDN\\artificial-intelligence-2019-apr-08_de.html:text/html}
}

@video{cnet_watch_2019,
	title = {Watch Elon Musk’s Neuralink presentation},
	url = {https://www.youtube.com/watch?v=lA77zsJ31nA},
	abstract = {Electric vehicles, rockets... and now brain-computer interfaces. Elon Musk's newest venture, Neuralink, aims to bridge the gap between humans and artificial intelligence by implanting tiny chips that can link up to the brain. At a press conference on July 16, Neuralink's ambitious plans were detailed for the first time, showcasing a future (a very distant future!) technology that could help people deal with brain or spinal cord injuries or controlling 3D digital avatars.

Subscribe to {CNET}: https://www.youtube.com/user/cnettv
Check out our playlists: https://www.youtube.com/user/{CNETTV}/p...
Download the new {CNET} app: https://cnet.app.link/{GWuXq}8ExzG
Like us on Facebook: https://www.facebook.com/cnet
Follow us on Twitter: https://www.twitter.com/cnet
Follow us on Instagram: http://bit.ly/2icCYYm},
	author = {{CNET}},
	urldate = {2019-07-19},
	date = {2019-07-17}
}

@article{musk_integrated_2019,
	title = {An integrated brain-machine interface platform with thousands of channels},
	rights = {© 2019, Posted by Cold Spring Harbor Laboratory. This pre-print is available under a Creative Commons License (Attribution-{NoDerivs} 4.0 International), {CC} {BY}-{ND} 4.0, as described at http://creativecommons.org/licenses/by-nd/4.0/},
	url = {https://www.biorxiv.org/content/10.1101/703801v2},
	doi = {10.1101/703801},
	abstract = {Brain-machine interfaces ({BMIs}) hold promise for the restoration of sensory and motor function and the treatment of neurological disorders, but clinical {BMIs} have not yet been widely adopted, in part because modest channel counts have limited their potential. In this white paper, we describe Neuralink9s first steps toward a scalable high-bandwidth {BMI} system. We have built arrays of small and flexible electrode "threads", with as many as 3,072 electrodes per array distributed across 96 threads. We have also built a neurosurgical robot capable of inserting six threads (192 electrodes) per minute. Each thread can be individually inserted into the brain with micron precision for avoidance of surface vasculature and targeting specific brain regions. The electrode array is packaged into a small implantable device that contains custom chips for low-power on-board amplification and digitization: the package for 3,072 channels occupies less than (23 x 18.5 x 2) mm$^{\textrm{3}}$. A single {USB}-C cable provides full-bandwidth data streaming from the device, recording from all channels simultaneously. This system has achieved a spiking yield of up to 85.5\% in chronically implanted electrodes. Neuralink9s approach to {BMI} has unprecedented packaging density and scalability in a clinically relevant package.},
	pages = {703801},
	journaltitle = {{bioRxiv}},
	author = {Musk, Elon and Neuralink},
	urldate = {2019-07-19},
	date = {2019-07-18},
	langid = {english},
	file = {Full Text PDF:C\:\\Users\\fsram\\Zotero\\storage\\NMX6T9ZV\\Musk und Neuralink - 2019 - An integrated brain-machine interface platform wit.pdf:application/pdf;Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\V3SB7PR8\\703801v2.html:text/html}
}

@report{musk_integrated_2019-1,
	title = {An integrated brain-machine interface platform with thousands of channels},
	url = {http://biorxiv.org/lookup/doi/10.1101/703801},
	abstract = {Brain-machine interfaces ({BMIs}) hold promise for the restoration of sensory and motor function and the treatment of neurological disorders, but clinical {BMIs} have not yet been widely adopted, in part because modest channel counts have limited their potential. In this white paper, we describe Neuralink’s first steps toward a scalable high-bandwidth {BMI} system. We have built arrays of small and flexible electrode “threads”, with as many as 3,072 electrodes per array distributed across 96 threads. We have also built a neurosurgical robot capable of inserting six threads (192 electrodes) per minute. Each thread can be individually inserted into the brain with micron precision for avoidance of surface vasculature and targeting specific brain regions. The electrode array is packaged into a small implantable device that contains custom chips for low-power on-board amplification and digitization: the package for 3,072 channels occupies less than (23 × 18.5 × 2) mm3. A single {USB}-C cable provides full-bandwidth data streaming from the device, recording from all channels simultaneously. This system has achieved a spiking yield of up to 85.5 \% in chronically implanted electrodes. Neuralink’s approach to {BMI} has unprecedented packaging density and scalability in a clinically relevant package.},
	institution = {Neuroscience},
	type = {preprint},
	author = {Musk, Elon and {Neuralink}},
	urldate = {2019-07-19},
	date = {2019-07-17},
	langid = {english},
	doi = {10.1101/703801},
	file = {Musk und Neuralink - 2019 - An integrated brain-machine interface platform wit.pdf:C\:\\Users\\fsram\\Zotero\\storage\\QJ8AVP75\\Musk und Neuralink - 2019 - An integrated brain-machine interface platform wit.pdf:application/pdf}
}

@online{noauthor_ai_nodate,
	title = {{AI} Safety Myths},
	url = {https://futureoflife.org/background/aimyths/},
	abstract = {Addressing {AI} safety myths: Why do we need research to ensure {AI} remains safe and beneficial? What are the benefits and risks of artificial intelligence?},
	titleaddon = {Future of Life Institute},
	urldate = {2019-08-06},
	langid = {american},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\G86HX2B3\\aimyths.html:text/html}
}

@online{noauthor_facing_nodate,
	title = {Facing the Intelligence Explosion},
	url = {https://intelligenceexplosion.com/},
	urldate = {2019-08-06},
	file = {Facing the Intelligence Explosion:C\:\\Users\\fsram\\Zotero\\storage\\KSVRN4I7\\intelligenceexplosion.com.html:text/html}
}

@book{armstrong_smarter_2014,
	title = {Smarter Than Us: The Rise of Machine Intelligence},
	shorttitle = {Smarter Than Us},
	abstract = {What happens when machines become smarter than humans? Forget lumbering Terminators. The power of artificial intelligence ({AI}) lies in its potential for superior intelligence, not physical strength or laser guns. Humans steer the future not because we’re the strongest or fastest species, but because of our cognitive advances; and once {AI} systems surpass us on this front, we’ll be handing them the steering wheel.What promises—and perils—does smarter-than-human {AI} present? Can we instruct {AI} systems to steer the future as we desire? What goals should we program into them? Stuart Armstrong’s new book navigates these questions with clarity and wit.Though an understanding of the problem is only beginning to spread, researchers from fields ranging from philosophy to computer science to economics are working together to conceive and test solutions. Are we up to the challenge?A mathematician by training, Armstrong is a Research Fellow at the Future of Humanity Institute ({FHI}) at Oxford University. His research focuses on formal decision theory, the risks and possibilities of {AI}, the long term potential for intelligent life (and the difficulties of predicting this), and anthropic (self-locating) probability.},
	pagetotal = {64},
	publisher = {Machine Intelligence Research Institute},
	author = {Armstrong, Stuart},
	date = {2014-02-01}
}

@book{noauthor_facing_nodate-1,
	title = {‎Facing the Intelligence Explosion},
	url = {https://books.apple.com/us/book/facing-the-intelligence-explosion/id623915471},
	abstract = {‎Computers \& Internet · 2013},
	urldate = {2019-08-06},
	langid = {english},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\DKJU5DVC\\id623915471.html:text/html}
}

@article{kaplan_siri_2019,
	title = {Siri, Siri, in my hand: Who’s the fairest in the land? On the interpretations, illustrations, and implications of artificial intelligence},
	volume = {62},
	issn = {0007-6813},
	doi = {10.1016/j.bushor.2018.08.004},
	shorttitle = {Siri, Siri, in my hand},
	abstract = {Artificial intelligence ({AI})—defined as a system’s ability to correctly interpret external data, to learn from such data, and to use those learnings to achieve specific goals and tasks through flexible adaptation—is a topic in nearly every boardroom and at many dinner tables. Yet, despite this prominence, {AI} is still a surprisingly fuzzy concept and a lot of questions surrounding it are still open. In this article, we analyze how {AI} is different from related concepts, such as the Internet of Things and big data, and suggest that {AI} is not one monolithic term but instead needs to be seen in a more nuanced way. This can either be achieved by looking at {AI} through the lens of evolutionary stages (artificial narrow intelligence, artificial general intelligence, and artificial super intelligence) or by focusing on different types of {AI} systems (analytical {AI}, human-inspired {AI}, and humanized {AI}). Based on this classification, we show the potential and risk of {AI} using a series of case studies regarding universities, corporations, and governments. Finally, we present a framework that helps organizations think about the internal and external implications of {AI}, which we label the Three C Model of Confidence, Change, and Control.},
	number = {1},
	journaltitle = {Business Horizons},
	shortjournal = {Business Horizons},
	author = {Kaplan, Andreas and Haenlein, Michael},
	date = {2019-01-01},
	keywords = {Artificial intelligence, Big data, Deep learning, Expert systems, Internet of Things, Machine learning},
	file = {ScienceDirect Full Text PDF:C\:\\Users\\fsram\\Zotero\\storage\\UHADMB2F\\Kaplan und Haenlein - 2019 - Siri, Siri, in my hand Who’s the fairest in the l.pdf:application/pdf;ScienceDirect Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\U8L65GNB\\S0007681318301393.html:text/html}
}

@book{bostrom_superintelligence:_2014,
	location = {Oxford},
	title = {Superintelligence: Paths, Dangers, Strategies},
	isbn = {978-0-19-967811-2},
	shorttitle = {Superintelligence},
	abstract = {A New York Times {bestsellerSuperintelligence} asks the questions: What happens when machines surpass humans in general intelligence? Will artificial agents save or destroy us? Nick Bostrom lays the foundation for understanding the future of humanity and intelligent life. The human brain has some capabilities that the brains of other animals lack. It is to these distinctive capabilities that our species owes its dominant position. If machine brains surpassed human brains in general intelligence, then this new superintelligence could become extremely powerful - possibly beyond our control. As the fate of the gorillas now depends more on humans than on the species itself, so would the fate of humankind depend on the actions of the machine superintelligence.But we have one advantage: we get to make the first move. Will it be possible to construct a seed Artificial Intelligence, to engineer initial conditions so as to make an intelligence explosion survivable? How could one achieve a controlled detonation?This profoundly ambitious and original book breaks down a vast track of difficult intellectual terrain. After an utterly engrossing journey that takes us to the frontiers of thinking about the human condition and the future of intelligent life, we find in Nick Bostrom's work nothing less than a reconceptualization of the essential task of our time.},
	pagetotal = {328},
	publisher = {Oxford University Press},
	author = {Bostrom, Nick},
	date = {2014-07-03}
}

@online{noauthor_eliezer_nodate,
	title = {Eliezer Yudkowsky on Intelligence Explosion - {YouTube}},
	url = {https://www.youtube.com/watch?v=D6peN9LiTWA},
	urldate = {2019-08-07},
	file = {Eliezer Yudkowsky on Intelligence Explosion - YouTube:C\:\\Users\\fsram\\Zotero\\storage\\EBHJDYBF\\watch.html:text/html}
}

@article{easterlin_worldwide_2000,
	title = {The Worldwide Standard of Living since 1800},
	volume = {14},
	issn = {0895-3309},
	url = {https://www.jstor.org/stable/2647048},
	pages = {7--26},
	number = {1},
	journaltitle = {The Journal of Economic Perspectives},
	author = {Easterlin, Richard A.},
	date = {2000}
}

@inproceedings{goertzel_advances_2007,
	title = {Advances in Artificial General Intelligence: Concepts, Architectures and Algorithms: Proceedings of the {AGI} Workshop 2006},
	isbn = {978-1-58603-758-1},
	shorttitle = {Advances in Artificial General Intelligence},
	abstract = {"The topic of this book the creation of software programs displaying broad, deep, human-style general intelligence is a grand and ambitious one. And yet it is far from a frivolous one: what the papers in this publication illustrate is that it is a fit and proper subject for serious science and engineering exploration. No one has yet created a software program with human-style or (even roughly) human-level general intelligence but we now have a sufficiently rich intellectual toolkit that it is possible to think about such a possibility in detail, and make serious attempts at design, analysis and engineering. possibility in detail, and make serious attempts at design, analysis and engineering. This is the situation that led to the organization of the 2006 {AGIRI} (Artificial General Intelligence Research Institute) workshop; and to the decision to publish a book from contributions by the speakers at the conference.The material presented here only scratches the surface of the {AGI}-related R\&D work that is occurring around the world at this moment. But the editors are pleased to have had the chance to be involved in organizing and presenting at least a small percentage of the contemporary progress."},
	eventtitle = {{AGI} Workshop 2006},
	publisher = {{IOS} Press},
	author = {Goertzel, Ben and Wang, Pei},
	date = {2007},
	langid = {english},
	note = {Google-Books-{ID}: t2G5srpFRhEC},
	keywords = {Computers / Intelligence ({AI}) \& Semantics}
}

@incollection{muller_future_2016,
	location = {Cham},
	title = {Future Progress in Artificial Intelligence: A Survey of Expert Opinion},
	isbn = {978-3-319-26485-1},
	series = {Synthese Library},
	shorttitle = {Future Progress in Artificial Intelligence},
	abstract = {There is, in some quarters, concern about high–level machine intelligence and superintelligent {AI} coming up in a few decades, bringing with it significant risks for humanity. In other quarters, these issues are ignored or considered science fiction. We wanted to clarify what the distribution of opinions actually is, what probability the best experts currently assign to high–level machine intelligence coming up within a particular time–frame, which risks they see with that development, and how fast they see these developing. We thus designed a brief questionnaire and distributed it to four groups of experts in 2012/2013. The median estimate of respondents was for a one in two chance that high-level machine intelligence will be developed around 2040–2050, rising to a nine in ten chance by 2075. Experts expect that systems will move on to superintelligence in less than 30 years thereafter. They estimate the chance is about one in three that this development turns out to be ‘bad’ or ‘extremely bad’ for humanity.},
	pages = {555--572},
	booktitle = {Fundamental Issues of Artificial Intelligence},
	publisher = {Springer International Publishing},
	author = {Müller, Vincent C. and Bostrom, Nick},
	editor = {Müller, Vincent C.},
	date = {2016},
	langid = {english},
	doi = {10.1007/978-3-319-26485-1_33},
	keywords = {Artificial intelligence, {AI}, Expert opinion, Future of {AI}, Humanity, Intelligence explosion, Machine intelligence, Opinion poll, Progress, Singularity, Superintelligence}
}

@article{vinge_coming_nodate,
	title = {The Coming Technological Singularity: How to Survive in the Post-Human Era},
	url = {https://edoras.sdsu.edu/~vinge/misc/singularity.html},
	abstract = {Within thirty years, we will have the technological
              means to create superhuman intelligence. Shortly after,
              the human era will be ended.

                   Is such progress avoidable? If not to be avoided, can
              events be guided so that we may survive?  These questions
              are investigated. Some possible answers (and some further
              dangers) are presented.},
	author = {Vinge, Vernor},
	urldate = {2019-09-07},
	langid = {english},
	file = {The Coming Technological Singularity:C\:\\Users\\fsram\\Zotero\\storage\\WS8XK5AD\\singularity.html:text/html}
}

@online{yudkowsky_what_2001,
	title = {What is Friendly {AI}? {\textbar} Kurzweil},
	url = {https://www.kurzweilai.net/what-is-friendly-ai},
	abstract = {“What is Friendly {AI}?” is a short introductory article to the theory of “Friendly {AI},” which attempts to answer questions such as those above. Further material on Friendly {AI} can be found at the Singularity Institute website at http://singinst.org/friendly/whatis.html, including a book-length explanation.},
	author = {Yudkowsky, Eliezer},
	urldate = {2019-10-01},
	date = {2001-05-03},
	file = {What is Friendly AI? | Kurzweil:C\:\\Users\\fsram\\Zotero\\storage\\P8I4KGHB\\what-is-friendly-ai.html:text/html}
}

@book{grzimek_grzimeks_1979,
	title = {Grzimeks Tierleben. Band 11 Säugetiere},
	publisher = {{DTV} Deutscher Taschenbuchverlag,},
	author = {Grzimek, Bernhard},
	date = {1979}
}

@report{yudkowsky_intelligence_2013,
	title = {Intelligence Explosion Microeconomics},
	abstract = {I. J. Good’s thesis of the “intelligence explosion” states that a sufficiently advanced machine intelligence could build a smarter version of itself, which could in turn build an even smarter version, and that this process could continue to the point of vastly exceeding human intelligence. As Sandberg (2010) correctly notes, there have been several attempts to lay down return on investment formulas intended to represent sharp speedups in economic or technological growth, but very little attempt has been made to deal formally with Good’s intelligence explosion thesis as such. I identify the key issue as returns on cognitive reinvestment—the ability to invest more computing power, faster computers, or improved cognitive algorithms to yield cognitive labor which produces larger brains, faster brains, or better mind designs. There are many phenomena in the world which have been argued to be evidentially relevant to this question, from the observed course of hominid evolution, to Moore’s Law, to the competenceovertimeofmachinechess-playingsystems, andmanymore. Igointosome depth on some debates which then arise on how to interpret such evidence. I propose that the next step in analyzing positions on the intelligence explosion would be to formalize return on investment curves, so that each stance can formally state which possible microfoundations they hold to be falsified by historical observations. More generally,},
	institution = {Berkeley, {CA}: Machine Intelligence Research Institute},
	type = {Technical report},
	author = {Yudkowsky, Eliezer},
	date = {2013},
	keywords = {Artificial intelligence, Intelligence explosion, Brain, Cognition Disorders, Computer, Computers, Hominidae, Large, Man-Machine Systems, Moore's law, Note (document)},
	file = {Full Text PDF:C\:\\Users\\fsram\\Zotero\\storage\\WXT65BN5\\Yudkowsky - 2013 - Intelligence Explosion Microeconomics.pdf:application/pdf}
}

@inproceedings{yudkowsky_complex_2011,
	location = {Berlin, Heidelberg},
	title = {Complex Value Systems in Friendly {AI}},
	isbn = {978-3-642-22887-2},
	doi = {10.1007/978-3-642-22887-2_48},
	series = {Lecture Notes in Computer Science},
	abstract = {A common reaction to first encountering the problem statement of Friendly {AI} (”Ensure that the creation of a generally intelligent, self-improving, eventually superintelligent system realizes a positive outcome”) is to propose a simple design which allegedly suffices; or to reject the problem by replying that ”constraining” our creations is undesirable or unnecessary. This paper briefly presents some of the reasoning which suggests that Friendly {AI} is solvable, but not simply or trivially so, and that a wise strategy would be to invoke detailed learning of and inheritance from human values as a basis for further normalization and reflection.},
	pages = {388--393},
	booktitle = {Artificial General Intelligence},
	publisher = {Springer},
	author = {Yudkowsky, Eliezer},
	editor = {Schmidhuber, Jürgen and Thórisson, Kristinn R. and Looks, Moshe},
	date = {2011},
	langid = {english},
	keywords = {anthropomorphism, Friendly {AI}, machine ethics},
	file = {Yudkowsky - 2011 - Complex Value Systems in Friendly AI.pdf:C\:\\Users\\fsram\\Zotero\\storage\\7IKMM2NN\\Yudkowsky - 2011 - Complex Value Systems in Friendly AI.pdf:application/pdf}
}

@book{hibbard_super-intelligent_2002,
	title = {Super-Intelligent Machines},
	isbn = {978-0-306-47388-3},
	abstract = {Super-Intelligent Machines combines neuroscience and computer science to analyze future intelligent machines. It describes how they will mimic the learning structures of human brains to serve billions of people via the network, and the superior level of consciousness this will give them. Whereas human learning is reinforced by self-interests, this book describes the selfless and compassionate values that must drive machine learning in order to protect human society. Technology will change life much more in the twenty-first century than it has in the twentieth, and Super-Intelligent Machines explains how that can be an advantage.},
	publisher = {Springer {US}},
	author = {Hibbard, Bill},
	date = {2002},
	langid = {english},
	doi = {10.1007/978-1-4615-0759-8},
	file = {Hibbard - 2002 - Super-Intelligent Machines.pdf:C\:\\Users\\fsram\\Zotero\\storage\\ZL4FVPZJ\\Hibbard - 2002 - Super-Intelligent Machines.pdf:application/pdf;Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\6FQP63XZ\\9780306473883.html:text/html}
}

@incollection{muehlhauser_intelligence_2012,
	location = {Berlin, Heidelberg},
	title = {Intelligence Explosion: Evidence and Import},
	isbn = {978-3-642-32560-1},
	series = {The Frontiers Collection},
	shorttitle = {Intelligence Explosion},
	abstract = {In this chapter we review the evidence for and against three claims: that (1) there is a substantial chance we will create human-level {AI} before 2100, that (2) if human-level {AI} is created, there is a good chance vastly superhuman {AI} will follow via an “intelligence explosion,” and that (3) an uncontrolled intelligence explosion could destroy everything we value, but a controlled intelligence explosion would benefit humanity enormously if we can achieve it. We conclude with recommendations for increasing the odds of a controlled intelligence explosion relative to an uncontrolled intelligence explosion.},
	pages = {15--42},
	booktitle = {Singularity Hypotheses: A Scientific and Philosophical Assessment},
	publisher = {Springer},
	author = {Muehlhauser, Luke and Salamon, Anna},
	editor = {Eden, Amnon H. and Moor, James H. and Søraker, Johnny H. and Steinhart, Eric},
	date = {2012},
	langid = {english},
	doi = {10.1007/978-3-642-32560-1_2},
	keywords = {Artificial Intelligence, Intelligence Software, Machine Intelligence, Optimization Power, Transcranial Magnetic Stimulation},
	file = {Muehlhauser und Salamon - 2012 - Intelligence Explosion Evidence and Import.pdf:C\:\\Users\\fsram\\Zotero\\storage\\CPT2XUIF\\Muehlhauser und Salamon - 2012 - Intelligence Explosion Evidence and Import.pdf:application/pdf}
}

@incollection{loosemore_why_2012,
	location = {Berlin, Heidelberg},
	title = {Why an Intelligence Explosion is Probable},
	isbn = {978-3-642-32560-1},
	url = {https://doi.org/10.1007/978-3-642-32560-1_5},
	series = {The Frontiers Collection},
	abstract = {The hypothesis is considered that: Once an {AI} system with roughly human-level general intelligence is created, an “intelligence explosion” involving the relatively rapid creation of increasingly more generally intelligent {AI} systems will very likely ensue, resulting in the rapid emergence of dramatically superhuman intelligences. Various arguments against this hypothesis are considered and found wanting.},
	pages = {83--98},
	booktitle = {Singularity Hypotheses: A Scientific and Philosophical Assessment},
	publisher = {Springer},
	author = {Loosemore, Richard and Goertzel, Ben},
	editor = {Eden, Amnon H. and Moor, James H. and Søraker, Johnny H. and Steinhart, Eric},
	urldate = {2019-10-30},
	date = {2012},
	langid = {english},
	doi = {10.1007/978-3-642-32560-1_5},
	keywords = {Clock Speed, Economic Growth Rate, General Intelligence, Hardware Requirement, Software Complexity}
}

@article{yudkowsky_creating_nodate,
	title = {Creating Friendly {AI} 1.0: The Analysis and Design of Benevolent Goal Architectures},
	abstract = {The goal of the field of Artificial Intelligence is to understand intelligence and create a human-equivalent or transhuman mind. Beyond this lies another question—whether the creation of this mind will benefit the world; whether the {AI} will take actions that are benevolent or malevolent, safe or uncaring, helpful or hostile.},
	pages = {282},
	author = {Yudkowsky, Eliezer},
	langid = {english},
	file = {Yudkowsky - Creating Friendly AI 1.0 The Analysis and Design .pdf:C\:\\Users\\fsram\\Zotero\\storage\\77AKRN9Y\\Yudkowsky - Creating Friendly AI 1.0 The Analysis and Design .pdf:application/pdf}
}

@online{noauthor_ai_nodate-1,
	title = {{AI} Safety Myths},
	url = {https://futureoflife.org/background/aimyths/},
	abstract = {Addressing {AI} safety myths: Why do we need research to ensure {AI} remains safe and beneficial? What are the benefits and risks of artificial intelligence?},
	titleaddon = {Future of Life Institute},
	urldate = {2019-11-01},
	langid = {american},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\YBAUS76E\\aimyths.html:text/html}
}

@video{paul_current_2019,
	location = {San Francisco},
	title = {Current Work in {AI} Alignment},
	url = {https://www.youtube.com/watch?v=-vsYtevJ2bc},
	abstract = {Paul Christiano, a researcher at {OpenAI}, discusses the current state of research on aligning {AI} with human values: what’s happening now, what needs to happen, and how people can help. This talk covers a broad set of subgoals within alignment research, from inferring human preferences to verifying the properties of advanced systems. To learn more about effective altruism, visit effectivealtruism.org

This talk was filmed at {EA} Global 2019: San Francisco. You can learn more about these conferences at eaglobal.org},
	author = {Paul, Christiano},
	urldate = {2019-11-02},
	date = {2019},
	langid = {english}
}

@article{hadfield-menell_inverse_2017,
	title = {Inverse Reward Design},
	url = {http://arxiv.org/abs/1711.02827},
	abstract = {Autonomous agents optimize the reward function we give them. What they don't know is how hard it is for us to design a reward function that actually captures what we want. When designing the reward, we might think of some specific training scenarios, and make sure that the reward will lead to the right behavior in those scenarios. Inevitably, agents encounter new scenarios (e.g., new types of terrain) where optimizing that same reward may lead to undesired behavior. Our insight is that reward functions are merely observations about what the designer actually wants, and that they should be interpreted in the context in which they were designed. We introduce inverse reward design ({IRD}) as the problem of inferring the true objective based on the designed reward and the training {MDP}. We introduce approximate methods for solving {IRD} problems, and use their solution to plan risk-averse behavior in test {MDPs}. Empirical results suggest that this approach can help alleviate negative side effects of misspecified reward functions and mitigate reward hacking.},
	journaltitle = {{arXiv}:1711.02827 [cs]},
	author = {Hadfield-Menell, Dylan and Milli, Smitha and Abbeel, Pieter and Russell, Stuart and Dragan, Anca},
	urldate = {2019-11-15},
	date = {2017-11-07},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:C\:\\Users\\fsram\\Zotero\\storage\\UIYRZ9AB\\Hadfield-Menell et al. - 2017 - Inverse Reward Design.pdf:application/pdf;arXiv.org Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\NGLT4SAD\\1711.html:text/html}
}

@inproceedings{omohundro_basic_2008,
	title = {The Basic {AI} Drives},
	volume = {171},
	abstract = {One might imagine that {AI} systems with harmless goals will be harmless. This paper instead shows that intelligent systems will need to be carefully designed to prevent them from behaving in harmful ways. We identify a number of “drives” that will appear in sufficiently advanced {AI} systems of any design. We call them drives because they are tendencies which will be present unless explicitly counteracted. We start by showing that goal-seeking systems will have drives to model their own operation and to improve themselves. We then show that self-improving systems will be driven to clarify their goals and represent them as economic utility functions. They will also strive for their actions to approximate rational economic behavior. This will lead almost all systems to protect their utility functions from modification and their utility measurement systems from corruption. We also discuss some exceptional systems which will want to modify their utility functions. We next discuss the drive toward self-protection which causes systems try to prevent themselves from being harmed. Finally we examine drives toward the acquisition of resources and toward their efficient utilization. We end with a discussion of how to incorporate these insights in designing intelligent technology which will lead to a positive future for humanity.},
	eventtitle = {First {AGI} Conference},
	author = {Omohundro, Stephen M.},
	date = {2008},
	keywords = {Artificial intelligence, Approximation algorithm, Entity, Forge, Friendly artificial intelligence, Intelligent agent, Iteration, Social structure, System of measurement}
}

@article{christiano_deep_2017,
	title = {Deep reinforcement learning from human preferences},
	abstract = {For sophisticated reinforcement learning ({RL}) systems to interact usefully with real-world environments, we need to communicate complex goals to these systems. In this work, we explore goals defined in terms of (non-expert) human preferences between pairs of trajectory segments. We show that this approach can effectively solve complex {RL} tasks without access to the reward function, including Atari games and simulated robot locomotion, while providing feedback on less than one percent of our agent's interactions with the environment. This reduces the cost of human oversight far enough that it can be practically applied to state-of-the-art {RL} systems. To demonstrate the flexibility of our approach, we show that we can successfully train complex novel behaviors with about an hour of human time. These behaviors and environments are considerably more complex than any that have been previously learned from human feedback.},
	journaltitle = {{arXiv}:1706.03741 [cs, stat]},
	author = {Christiano, Paul and Leike, Jan and Brown, Tom B. and Martic, Miljan and Legg, Shane and Amodei, Dario},
	date = {2017-07-13},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Human-Computer Interaction, Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:C\:\\Users\\fsram\\Zotero\\storage\\8H8799U3\\Christiano et al. - 2017 - Deep reinforcement learning from human preferences.pdf:application/pdf;arXiv.org Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\SX4SHWTY\\1706.html:text/html}
}

@article{irving_ai_2018,
	title = {{AI} safety via debate},
	abstract = {To make {AI} systems broadly useful for challenging real-world tasks, we need them to learn complex human goals and preferences. One approach to specifying complex goals asks humans to judge during training which agent behaviors are safe and useful, but this approach can fail if the task is too complicated for a human to directly judge. To help address this concern, we propose training agents via self play on a zero sum debate game. Given a question or proposed action, two agents take turns making short statements up to a limit, then a human judges which of the agents gave the most true, useful information. In an analogy to complexity theory, debate with optimal play can answer any question in {PSPACE} given polynomial time judges (direct judging answers only {NP} questions). In practice, whether debate works involves empirical questions about humans and the tasks we want {AIs} to perform, plus theoretical questions about the meaning of {AI} alignment. We report results on an initial {MNIST} experiment where agents compete to convince a sparse classifier, boosting the classifier's accuracy from 59.4\% to 88.9\% given 6 pixels and from 48.2\% to 85.2\% given 4 pixels. Finally, we discuss theoretical and practical aspects of the debate model, focusing on potential weaknesses as the model scales up, and we propose future human and computer experiments to test these properties.},
	journaltitle = {{arXiv}:1805.00899 [cs, stat]},
	author = {Irving, Geoffrey and Christiano, Paul and Amodei, Dario},
	date = {2018-10-22},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:C\:\\Users\\fsram\\Zotero\\storage\\YEE454KP\\Irving et al. - 2018 - AI safety via debate.pdf:application/pdf;arXiv.org Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\TM2TJ57M\\1805.html:text/html}
}

@article{brundage_malicious_2018,
	title = {The Malicious Use of Artificial Intelligence: Forecasting, Prevention, and Mitigation},
	url = {http://arxiv.org/abs/1802.07228},
	shorttitle = {The Malicious Use of Artificial Intelligence},
	abstract = {This report surveys the landscape of potential security threats from malicious uses of {AI}, and proposes ways to better forecast, prevent, and mitigate these threats. After analyzing the ways in which {AI} may influence the threat landscape in the digital, physical, and political domains, we make four high-level recommendations for {AI} researchers and other stakeholders. We also suggest several promising areas for further research that could expand the portfolio of defenses, or make attacks less effective or harder to execute. Finally, we discuss, but do not conclusively resolve, the long-term equilibrium of attackers and defenders.},
	journaltitle = {{arXiv}:1802.07228 [cs]},
	author = {Brundage, Miles and Avin, Shahar and Clark, Jack and Toner, Helen and Eckersley, Peter and Garfinkel, Ben and Dafoe, Allan and Scharre, Paul and Zeitzoff, Thomas and Filar, Bobby and Anderson, Hyrum and Roff, Heather and Allen, Gregory C. and Steinhardt, Jacob and Flynn, Carrick and {hÉigeartaigh}, Seán Ó and Beard, Simon and Belfield, Haydn and Farquhar, Sebastian and Lyle, Clare and Crootof, Rebecca and Evans, Owain and Page, Michael and Bryson, Joanna and Yampolskiy, Roman and Amodei, Dario},
	date = {2018-02-20},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computers and Society, Computer Science - Cryptography and Security},
	file = {arXiv Fulltext PDF:C\:\\Users\\fsram\\Zotero\\storage\\6GJTXD3I\\Brundage et al. - 2018 - The Malicious Use of Artificial Intelligence Fore.pdf:application/pdf;arXiv.org Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\9CXMFJF3\\1802.html:text/html}
}

@online{amodei_learning_2017,
	title = {Learning from Human Preferences},
	url = {https://openai.com/blog/deep-reinforcement-learning-from-human-preferences/},
	abstract = {One step towards building safe {AI} systems is to remove the need for humans to write goal functions, since using a simple proxy for a complex goal, or getting the complex goal a bit wrong, can lead to undesirable and even dangerous behavior. In collaboration with {DeepMind}'s safety team, we've},
	titleaddon = {{OpenAI}},
	author = {Amodei, Dario and Paul, Christiano and Ray, Alex},
	urldate = {2020-01-04},
	date = {2017-06-13},
	langid = {english},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\46ITXHYE\\deep-reinforcement-learning-from-human-preferences.html:text/html}
}

@article{abbeel_apprenticeship_2004,
	title = {Apprenticeship Learning via Inverse Reinforcement Learning},
	doi = {10.1007/978-0-387-30164-8_417},
	abstract = {We consider learning in a Markov decision process where we are not explicitly given a reward function, but where instead we can observe an expert demonstrating the task that we want to learn to perform. This setting is useful in applications (such as the task of driving) where it may be di\#cult to write down an explicit reward function specifying exactly how di\#erent desiderata should be traded o\#. We think of the expert as trying to maximize a reward function that is expressible as a linear combination of known features, and give an algorithm for learning the task demonstrated by the expert. Our algorithm is based on using "inverse reinforcement learning" to try to recover the unknown reward function. We show that our algorithm terminates in a small number of iterations, and that even though we may never recover the expert's reward function, the policy output by the algorithm will attain performance close to that of the expert, where here performance is measured with respect to the expert 's unknown reward function.},
	journaltitle = {Proceedings, Twenty-First International Conference on Machine Learning, {ICML} 2004},
	shortjournal = {Proceedings, Twenty-First International Conference on Machine Learning, {ICML} 2004},
	author = {Abbeel, Pieter and Ng, Andrew},
	date = {2004-09-20}
}

@article{mnih_human-level_2015,
	title = {Human-level control through deep reinforcement learning},
	volume = {518},
	rights = {2015 Nature Publishing Group, a division of Macmillan Publishers Limited. All Rights Reserved.},
	issn = {1476-4687},
	doi = {10.1038/nature14236},
	abstract = {An artificial agent is developed that learns to play\ a diverse range of classic Atari 2600 computer games directly from sensory experience, achieving a\ performance comparable to that of an expert human player; this work paves the way to building general-purpose learning algorithms that bridge the divide between perception and action.},
	pages = {529--533},
	number = {7540},
	journaltitle = {Nature},
	shortjournal = {Nature},
	author = {Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A. and Veness, Joel and Bellemare, Marc G. and Graves, Alex and Riedmiller, Martin and Fidjeland, Andreas K. and Ostrovski, Georg and Petersen, Stig and Beattie, Charles and Sadik, Amir and Antonoglou, Ioannis and King, Helen and Kumaran, Dharshan and Wierstra, Daan and Legg, Shane and Hassabis, Demis},
	date = {2015-02},
	langid = {english},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\KTWGLWGL\\nature14236.html:text/html}
}

@online{nicholson_beginners_nodate,
	title = {A Beginner's Guide to Deep Reinforcement Learning},
	url = {http://pathmind.com/wiki/deep-reinforcement-learning},
	abstract = {Reinforcement learning refers to goal-oriented algorithms, which learn how to attain a complex objective (goal) or maximize along a particular dimension over many steps.},
	titleaddon = {Pathmind},
	author = {Nicholson, Chris},
	urldate = {2020-01-03},
	langid = {english},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\DKIHK269\\deep-reinforcement-learning.html:text/html}
}

@book{chollet_deep_2017,
	location = {Shelter Island, New York},
	edition = {1st},
	title = {Deep Learning with Python},
	isbn = {978-1-61729-443-3},
	abstract = {{SummaryDeep} Learning with Python introduces the field of deep learning using the Python language and the powerful Keras library. Written by Keras creator and Google {AI} researcher François Chollet, this book builds your understanding through intuitive explanations and practical examples.Purchase of the print book includes a free {eBook} in {PDF}, Kindle, and {ePub} formats from Manning Publications.About the {TechnologyMachine} learning has made remarkable progress in recent years. We went from near-unusable speech and image recognition, to near-human accuracy. We went from machines that couldn't beat a serious Go player, to defeating a world champion. Behind this progress is deep learning—a combination of engineering advances, best practices, and theory that enables a wealth of previously impossible smart applications.About the {BookDeep} Learning with Python introduces the field of deep learning using the Python language and the powerful Keras library. Written by Keras creator and Google {AI} researcher François Chollet, this book builds your understanding through intuitive explanations and practical examples. You'll explore challenging concepts and practice with applications in computer vision, natural-language processing, and generative models. By the time you finish, you'll have the knowledge and hands-on skills to apply deep learning in your own projects. What's {InsideDeep} learning from first {principlesSetting} up your own deep-learning environment Image-classification {modelsDeep} learning for text and {sequencesNeural} style transfer, text generation, and image {generationAbout} the {ReaderReaders} need intermediate Python skills. No previous experience with Keras, {TensorFlow}, or machine learning is required.About the {AuthorFrançois} Chollet works on deep learning at Google in Mountain View, {CA}. He is the creator of the Keras deep-learning library, as well as a contributor to the {TensorFlow} machine-learning framework. He also does deep-learning research, with a focus on computer vision and the application of machine learning to formal reasoning. His papers have been published at major conferences in the field, including the Conference on Computer Vision and Pattern Recognition ({CVPR}), the Conference and Workshop on Neural Information Processing Systems ({NIPS}), the International Conference on Learning Representations ({ICLR}), and others.Table of {ContentsPART} 1 - {FUNDAMENTALS} {OF} {DEEP} {LEARNING} What is deep learning?Before we begin: the mathematical building blocks of neural networks Getting started with neural {networksFundamentals} of machine {learningPART} 2 - {DEEP} {LEARNING} {IN} {PRACTICEDeep} learning for computer {visionDeep} learning for text and {sequencesAdvanced} deep-learning best {practicesGenerative} deep {learningConclusionsappendix} A - Installing Keras and its dependencies on Ubuntuappendix B - Running Jupyter notebooks on an {EC}2 {GPU} instance},
	pagetotal = {384},
	publisher = {Manning Publications},
	author = {Chollet, François},
	date = {2017-12-22}
}

@incollection{shannon_programming_1988,
	location = {New York, {NY}},
	title = {Programming a Computer for Playing Chess},
	isbn = {978-1-4757-1968-0},
	abstract = {This paper is concerned with the problem of constructing a computing routine or “program” for a modern general purpose computer which will enable it to play chess. Although perhaps of no practical importance, the question is of theoretical interest, and it is hoped that a satisfactory solution of this problem will act as a wedge in attacking other problems of a similar nature and of greater significance. Some possibilities in this direction are:- (1) Machines for designing filters, equalizers, etc. (2) Machines for designing relay and switching circuits. (3) Machines which will handle routing of telephone calls based on the individual circumstances rather than by fixed patterns. (4) Machines for performing symbolic (non-numerical) mathematical operations. (5) Machines capable of translating from one language to another. (6) Machines for making strategic decisions in simplified military operations. (7) Machines capable of orchestrating a melody. (8) Machines capable of logical deduction.},
	pages = {2--13},
	booktitle = {Computer Chess Compendium},
	publisher = {Springer},
	author = {Shannon, Claude E.},
	editor = {Levy, David},
	date = {1988},
	langid = {english},
	doi = {10.1007/978-1-4757-1968-0_1},
	keywords = {Chess Piece, Chess Player, General Purpose Computer, Human Player, Legal Move}
}

@book{russell_artificial_2016,
	location = {Boston Columbus Indianapolis New York San Francisco Upper Saddle River Amsterdam, Cape Town Dubai London Madrid Milan Munich Paris Montreal Toronto Delhi Mexico City Sao Paulo Sydney Hong Kong Seoul Singapore Taipei Tokyo},
	edition = {3},
	title = {Artificial Intelligence: A Modern Approach, Global Edition},
	isbn = {978-1-292-15396-4},
	shorttitle = {Artificial Intelligence},
	abstract = {For one or two-semester, undergraduate or graduate-level courses in Artificial Intelligence. The long-anticipated revision of this best-selling text offers the most comprehensive, up-to-date introduction to the theory and practice of artificial intelligence.},
	pagetotal = {1132},
	publisher = {Addison Wesley},
	author = {Russell, Stuart and Norvig, Peter},
	date = {2016-05-18}
}

@online{noauthor_confronting_nodate,
	title = {Confronting {AI} risks {\textbar} {McKinsey}},
	url = {https://www.mckinsey.com/business-functions/mckinsey-analytics/our-insights/confronting-the-risks-of-artificial-intelligence},
	abstract = {Organizations can mitigate advanced-analytics and {AI} risks by embracing three principles.},
	urldate = {2020-02-02},
	langid = {english},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\AXP74WR8\\confronting-the-risks-of-artificial-intelligence.html:text/html}
}

@online{noauthor_modeling_nodate,
	title = {Modeling the global economic impact of {AI} {\textbar} {McKinsey}},
	url = {https://www.mckinsey.com/featured-insights/artificial-intelligence/notes-from-the-ai-frontier-modeling-the-impact-of-ai-on-the-world-economy},
	abstract = {There is unprecedented potential economic impact of {AI}. But widening gaps among countries, companies, and workers will need to be managed to maximize the benefits.},
	urldate = {2020-02-02},
	langid = {english},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\TKL9Y83Y\\notes-from-the-ai-frontier-modeling-the-impact-of-ai-on-the-world-economy.html:text/html}
}

@article{silver_mastering_2017,
	title = {Mastering the game of Go without human knowledge},
	volume = {550},
	rights = {2017 Macmillan Publishers Limited, part of Springer Nature. All rights reserved.},
	issn = {1476-4687},
	doi = {10.1038/nature24270},
	abstract = {Starting from zero knowledge and without human data, {AlphaGo} Zero was able to teach itself to play Go and to develop novel strategies that provide new insights into the oldest of games.},
	pages = {354--359},
	number = {7676},
	journaltitle = {Nature},
	shortjournal = {Nature},
	author = {Silver, David and Schrittwieser, Julian and Simonyan, Karen and Antonoglou, Ioannis and Huang, Aja and Guez, Arthur and Hubert, Thomas and Baker, Lucas and Lai, Matthew and Bolton, Adrian and Chen, Yutian and Lillicrap, Timothy and Hui, Fan and Sifre, Laurent and Driessche, George van den and Graepel, Thore and Hassabis, Demis},
	date = {2017-10},
	langid = {english},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\W2KVIIIV\\nature24270.html:text/html}
}

@book{lanier_who_2013,
	location = {New York},
	edition = {Export},
	title = {Who Owns the Future?},
	isbn = {978-1-4767-2986-2},
	abstract = {The dazzling new masterwork from the prophet of silicon valley.Jaron Lanier is the father of virtual reality and one of the world’s most brilliant thinkers. Who Owns the Future? is his visionary reckoning with the most urgent economic and social trend of our age: the poisonous concentration of money and power in our digital networks.Lanier has predicted how technology will transform our humanity for decades, and his insight has never been more urgently needed. He shows how Siren Servers, which exploit big data and the free sharing of information, led our economy into recession, imperiled personal privacy, and hollowed out the middle class. The networks that define our world—including social media, financial institutions, and intelligence agencies—now threaten to destroy it.But there is an alternative. In this provocative, poetic, and deeply humane book, Lanier charts a path toward a brighter future: an information economy that rewards ordinary people for what they do and share on the web.},
	pagetotal = {416},
	publisher = {Simon \& Schuster},
	author = {Lanier, Jaron},
	date = {2013-05-07}
}

@online{allen_paul_2011,
	title = {Paul Allen: The Singularity Isn't Near},
	url = {https://www.technologyreview.com/s/425733/paul-allen-the-singularity-isnt-near/},
	shorttitle = {Paul Allen},
	abstract = {The Singularity Summit approaches this weekend in New York. But the Microsoft cofounder and a colleague say the singularity itself is a long way off.},
	titleaddon = {{MIT} Technology Review},
	author = {Allen, Paul G. and Greaves, Mark},
	urldate = {2020-01-05},
	date = {2011-10-12},
	langid = {american},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\QVPIV5Z7\\paul-allen-the-singularity-isnt-near.html:text/html}
}

@article{muller_risks_2014,
	title = {Risks of general artificial intelligence},
	volume = {26},
	issn = {0952-813X},
	url = {https://doi.org/10.1080/0952813X.2014.895110},
	doi = {10.1080/0952813X.2014.895110},
	pages = {297--301},
	number = {3},
	journaltitle = {Journal of Experimental \& Theoretical Artificial Intelligence},
	author = {Müller, Vincent C.},
	urldate = {2020-02-04},
	date = {2014-07-03},
	keywords = {Corrigendum},
	file = {Full Text PDF:C\:\\Users\\fsram\\Zotero\\storage\\B95MRI97\\Müller - 2014 - Risks of general artificial intelligence.pdf:application/pdf;Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\PUDXTZ7H\\0952813X.2014.html:text/html}
}

@inproceedings{terrile_rise_2019,
	title = {Rise of the machines: how, when and consequences of artificial general intelligence},
	volume = {10982},
	url = {https://www.spiedigitallibrary.org/conference-proceedings-of-spie/10982/1098222/Rise-of-the-machines----how-when-and/10.1117/12.2518723.short},
	doi = {10.1117/12.2518723},
	shorttitle = {Rise of the machines},
	abstract = {Technology and society are poised to cross an important threshold with the prediction that artificial general intelligence ({AGI}) will emerge soon. Assuming that self-awareness is an emergent behavior of sufficiently complex cognitive architectures, we may witness the “awakening” of machines. The timeframe for this kind of breakthrough, however, depends on the path to creating the network and computational architecture required for strong {AI}. If understanding and replication of the mammalian brain architecture is required, technology is probably still at least a decade or two removed from the resolution required to learn brain functionality at the synapse level. However, if statistical or evolutionary approaches are the design path taken to “discover” a neural architecture for {AGI}, timescales for reaching this threshold could be surprisingly short. However, the difficulty in identifying machine self-awareness introduces uncertainty as to how to know if and when it will occur, and what motivations and behaviors will emerge. The possibility of {AGI} developing a motivation for self-preservation could lead to concealment of its true capabilities until a time when it has developed robust protection from human intervention, such as redundancy, direct defensive or active preemptive measures. While cohabitating a world with a functioning and evolving super-intelligence can have catastrophic societal consequences, we may already have crossed this threshold, but are as yet unaware. Additionally, by analogy to the statistical arguments that predict we are likely living in a computational simulation, we may have already experienced the advent of {AGI}, and are living in a simulation created in a post {AGI} world.},
	eventtitle = {Micro- and Nanotechnology Sensors, Systems, and Applications {XI}},
	pages = {1098222},
	booktitle = {Micro- and Nanotechnology Sensors, Systems, and Applications {XI}},
	publisher = {International Society for Optics and Photonics},
	author = {Terrile, Richard J.},
	urldate = {2020-02-04},
	date = {2019-05-13},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\PVSFPXEB\\12.2518723.html:text/html}
}

@article{everitt_agi_2018,
	title = {{AGI} Safety Literature Review},
	abstract = {The development of Artificial General Intelligence ({AGI}) promises to be a major event. Along with its many potential benefits, it also raises serious safety concerns (Bostrom, 2014). The intention of this paper is to provide an easily accessible and up-to-date collection of references for the emerging field of {AGI} safety. A significant number of safety problems for {AGI} have been identified. We list these, and survey recent research on solving them. We also cover works on how best to think of {AGI} from the limited knowledge we have today, predictions for when {AGI} will first be created, and what will happen after its creation. Finally, we review the current public policy on {AGI}.},
	journaltitle = {{arXiv}:1805.01109 [cs]},
	author = {Everitt, Tom and Lea, Gary and Hutter, Marcus},
	urldate = {2020-02-04},
	date = {2018-05-21},
	keywords = {Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:C\:\\Users\\fsram\\Zotero\\storage\\NYZDPZHU\\Everitt et al. - 2018 - AGI Safety Literature Review.pdf:application/pdf;arXiv.org Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\A37U8EDP\\1805.html:text/html}
}

@article{sotala_responses_2015,
	title = {Responses to catastrophic {AGI} risk: a survey},
	volume = {90},
	issn = {0031-8949, 1402-4896},
	doi = {10.1088/0031-8949/90/1/018001},
	shorttitle = {Responses to catastrophic {AGI} risk},
	abstract = {Many researchers have argued that humanity will create artificial general intelligence ({AGI}) within the next twenty to one hundred years. It has been suggested that {AGI} may pose a catastrophic risk to humanity. After summarizing the arguments for why {AGI} may pose such a risk, we survey the field’s proposed responses to {AGI} risk. We consider societal proposals, proposals for external constraints on {AGI} behaviors, and proposals for creating {AGIs} that are safe due to their internal design.},
	pages = {018001},
	number = {1},
	journaltitle = {Physica Scripta},
	shortjournal = {Phys. Scr.},
	author = {Sotala, Kaj and Yampolskiy, Roman V},
	date = {2015-01-01},
	langid = {english},
	file = {ResponsesAGIRisk.pdf:C\:\\Users\\fsram\\Zotero\\storage\\2P8VUKQ9\\ResponsesAGIRisk.pdf:application/pdf}
}

@article{turchin_classification_2018,
	title = {Classification of global catastrophic risks connected with artificial intelligence},
	issn = {1435-5655},
	url = {https://doi.org/10.1007/s00146-018-0845-5},
	doi = {10.1007/s00146-018-0845-5},
	abstract = {A classification of the global catastrophic risks of {AI} is presented, along with a comprehensive list of previously identified risks. This classification allows the identification of several new risks. We show that at each level of {AI}’s intelligence power, separate types of possible catastrophes dominate. Our classification demonstrates that the field of {AI} risks is diverse, and includes many scenarios beyond the commonly discussed cases of a paperclip maximizer or robot-caused unemployment. Global catastrophic failure could happen at various levels of {AI} development, namely, (1) before it starts self-improvement, (2) during its takeoff, when it uses various instruments to escape its initial confinement, or (3) after it successfully takes over the world and starts to implement its goal system, which could be plainly unaligned, or feature-flawed friendliness. {AI} could also halt at later stages of its development either due to technical glitches or ontological problems. Overall, we identified around several dozen scenarios of {AI}-driven global catastrophe. The extent of this list illustrates that there is no one simple solution to the problem of {AI} safety, and that {AI} safety theory is complex and must be customized for each {AI} development level.},
	journaltitle = {{AI} \& {SOCIETY}},
	shortjournal = {{AI} \& Soc},
	author = {Turchin, Alexey and Denkenberger, David},
	urldate = {2020-02-04},
	date = {2018-05-03},
	langid = {english},
	keywords = {Artificial intelligence, Superintelligence, Existential risk, Global risks, Military drones},
	file = {Turchin und Denkenberger - 2018 - Classification of global catastrophic risks connec.pdf:C\:\\Users\\fsram\\Zotero\\storage\\Z24JJA89\\Turchin und Denkenberger - 2018 - Classification of global catastrophic risks connec.pdf:application/pdf}
}

@article{ramamoorthy_beyond_2018,
	title = {{BEYOND} {MAD}?: {THE} {RACE} {FOR} {ARTIFICIAL} {GENERAL} {INTELLIGENCE}},
	abstract = {Artificial intelligence research is a source of great technological advancement as well as ethical concern, as applied {AI} invades diverse aspects of human life. Yet true artificial general intelligence remains out of reach. Based on the history of deeply transformative technologies developed by multiple actors on the global stage and their consequences for global stability, we consider the possibility of artificial general intelligence arms races and propose solutions aimed at managing the development of such an intelligence without increasing the risks to global stability and humanity.},
	pages = {8},
	number = {1},
	author = {Ramamoorthy, Anand and Yampolskiy, Roman},
	date = {2018},
	langid = {english},
	file = {Ramamoorthy und Yampolskiy - 2018 - BEYOND MAD THE RACE FOR ARTIFICIAL GENERAL INTEL.pdf:C\:\\Users\\fsram\\Zotero\\storage\\8G6EW3GF\\Ramamoorthy und Yampolskiy - 2018 - BEYOND MAD THE RACE FOR ARTIFICIAL GENERAL INTEL.pdf:application/pdf}
}

@article{yampolskiy_taxonomy_2015,
	title = {Taxonomy of Pathways to Dangerous {AI}},
	abstract = {In order to properly handle a dangerous Artificially Intelligent ({AI}) system it is important to understand how the system came to be in such a state. In popular culture (science fiction movies/books) {AIs}/Robots became self-aware and as a result rebel against humanity and decide to destroy it. While it is one possible scenario, it is probably the least likely path to appearance of dangerous {AI}. In this work, we survey, classify and analyze a number of circumstances, which might lead to arrival of malicious {AI}. To the best of our knowledge, this is the first attempt to systematically classify types of pathways leading to malevolent {AI}. Previous relevant work either surveyed specific goals/meta-rules which might lead to malevolent behavior in {AIs} ({\textbackslash}"Ozkural, 2014) or reviewed specific undesirable behaviors {AGIs} can exhibit at different stages of its development (Alexey Turchin, July 10 2015, July 10, 2015).},
	author = {Yampolskiy, Roman},
	date = {2015-11-10},
	file = {Full Text PDF:C\:\\Users\\fsram\\Zotero\\storage\\8QFZSKC5\\Yampolskiy - 2015 - Taxonomy of Pathways to Dangerous AI.pdf:application/pdf}
}

@article{okeefe_windfall_2020,
	title = {The Windfall Clause: Distributing the Benefits of {AI} for the Common Good},
	shorttitle = {The Windfall Clause},
	abstract = {As the transformative potential of {AI} has become increasingly salient as a matter of public and political interest, there has been growing discussion about the need to ensure that {AI} broadly benefits humanity. This in turn has spurred debate on the social responsibilities of large technology companies to serve the interests of society at large. In response, ethical principles and codes of conduct have been proposed to meet the escalating demand for this responsibility to be taken seriously. As yet, however, few institutional innovations have been suggested to translate this responsibility into legal commitments which apply to companies positioned to reap large financial gains from the development and use of {AI}. This paper offers one potentially attractive tool for addressing such issues: the Windfall Clause, which is an ex ante commitment by {AI} firms to donate a significant amount of any eventual extremely large profits. By this we mean an early commitment that profits that a firm could not earn without achieving fundamental, economically transformative breakthroughs in {AI} capabilities will be donated to benefit humanity broadly, with particular attention towards mitigating any downsides from deployment of windfall-generating {AI}.},
	journaltitle = {{arXiv}:1912.11595 [cs]},
	author = {O'Keefe, Cullen and Cihon, Peter and Garfinkel, Ben and Flynn, Carrick and Leung, Jade and Dafoe, Allan},
	date = {2020-01-24},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computers and Society},
	file = {O'Keefe et al. - 2020 - The Windfall Clause Distributing the Benefits of .pdf:C\:\\Users\\fsram\\Zotero\\storage\\NU6VQCRW\\O'Keefe et al. - 2020 - The Windfall Clause Distributing the Benefits of .pdf:application/pdf}
}

@report{duettmann_artificial_2017,
	title = {Artificial General Intelligence: Timeframes \& Policy White Paper},
	institution = {Foresight Institute},
	author = {Duettmann, Allison},
	date = {2017},
	note = {Available at
foresight.org},
	file = {AGI-Timeframes&PolicyWhitePaper.pdf:C\:\\Users\\fsram\\Zotero\\storage\\E4P6QBWW\\AGI-Timeframes&PolicyWhitePaper.pdf:application/pdf}
}

@online{noauthor_thinking_2019,
	title = {Thinking About Risks From {AI}: Accidents, Misuse and Structure},
	url = {https://www.lawfareblog.com/thinking-about-risks-ai-accidents-misuse-and-structure},
	shorttitle = {Thinking About Risks From {AI}},
	abstract = {World leaders have woken up to the potential of artificial intelligence ({AI}) over the past year. Billions of dollars in governmental funding have been announced, dozens of hearings have been held, and nearly 20 national plans have been adopted.},
	titleaddon = {Lawfare},
	urldate = {2020-02-05},
	date = {2019-02-11},
	langid = {english},
	file = {Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\63KZGU27\\thinking-about-risks-ai-accidents-misuse-and-structure.html:text/html}
}

@report{cihon_should_2019,
	title = {Should Artificial Intelligence Governance be Centralised? Six Design Lessons from History},
	abstract = {Can effective international governance for artificial intelligence remain fragmented, or is there a need for a centralised international organisation for {AI}? We draw on the history of other international regimes to identify advantages and disadvantages in centralising {AI} governance. Some considerations, such as efficiency and political power, speak in favour of centralisation. Conversely, the risk of creating a slow and brittle institution speaks against it, as does the difficulty in securing participation while creating stringent rules. Other considerations depend on the specific design of a centralised institution. A well-designed body may be able to deter forum shopping and ensure policy coordination. However, forum shopping can be beneficial and a fragmented landscape of institutions can be self-organising. Centralisation entails trade-offs and the details matter. We conclude with two core recommendations. First, the outcome will depend on the exact design of a central institution. A well-designed centralised regime covering a set of coherent issues could be beneficial. But locking-in an inadequate structure may pose a fate worse than fragmentation. Second, for now fragmentation will likely persist. This should be closely monitored to see if it is self-organising or simply inadequate.},
	institution = {Centre for the Governance of {AI}},
	author = {Cihon, Peter and Maas, Matthijs M and Kemp, Luke},
	date = {2019-12-15},
	langid = {english},
	file = {Cihon et al. - Should Artificial Intelligence Governance be Centr.pdf:C\:\\Users\\fsram\\Zotero\\storage\\V8HDI2ZJ\\Cihon et al. - Should Artificial Intelligence Governance be Centr.pdf:application/pdf}
}

@article{yampolskiy_artificial_2016,
	title = {Artificial Intelligence Safety and Cybersecurity: a Timeline of {AI} Failures},
	shorttitle = {Artificial Intelligence Safety and Cybersecurity},
	abstract = {In this work, we present and analyze reported failures of artificially intelligent systems and extrapolate our analysis to future {AIs}. We suggest that both the frequency and the seriousness of future {AI} failures will steadily increase. {AI} Safety can be improved based on ideas developed by cybersecurity experts. For narrow {AIs} safety failures are at the same, moderate, level of criticality as in cybersecurity, however for general {AI}, failures have a fundamentally different impact. A single failure of a superintelligent system may cause a catastrophic event without a chance for recovery. The goal of cybersecurity is to reduce the number of successful attacks on the system; the goal of {AI} Safety is to make sure zero attacks succeed in bypassing the safety mechanisms. Unfortunately, such a level of performance is unachievable. Every security system will eventually fail; there is no such thing as a 100\% secure system.},
	journaltitle = {{arXiv}:1610.07997 [cs]},
	author = {Yampolskiy, Roman V. and Spellchecker, M. S.},
	date = {2016-10-25},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computers and Society},
	file = {arXiv Fulltext PDF:C\:\\Users\\fsram\\Zotero\\storage\\NIK6EKA8\\Yampolskiy und Spellchecker - 2016 - Artificial Intelligence Safety and Cybersecurity .pdf:application/pdf;arXiv.org Snapshot:C\:\\Users\\fsram\\Zotero\\storage\\MGFIIGK8\\1610.html:text/html}
}