-
Notifications
You must be signed in to change notification settings - Fork 7
/
index12.html
638 lines (598 loc) · 57.9 KB
/
index12.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
<!DOCTYPE html>
<head>
<meta charset="utf-8">
<title>Store Halfword Byte-Reverse Indexed</title>
<meta name="author" content="OzLabs">
<link href="https://sthbrx.github.io/rss.xml" type="application/rss+xml" rel="alternate"
title="Store Halfword Byte-Reverse Indexed RSS Feed" />
<!-- http://t.co/dKP3o1e -->
<meta name="HandheldFriendly" content="True">
<meta name="MobileOptimized" content="320">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link href="https://sthbrx.github.io/favicon.png" rel="icon">
<link href="https://sthbrx.github.io/theme/css/main.css" media="screen, projection"
rel="stylesheet" type="text/css">
<link href="//fonts.googleapis.com/css?family=PT+Serif:regular,italic,bold,bolditalic"
rel="stylesheet" type="text/css">
<link href="//fonts.googleapis.com/css?family=PT+Sans:regular,italic,bold,bolditalic"
rel="stylesheet" type="text/css">
<script type="text/javascript">
document.addEventListener('DOMContentLoaded', function() {
var ts = document.createElement('span')
ts.className = 'toggle-sidebar'
ts = document.getElementById('content').appendChild(ts);
ts.addEventListener('click', function(e) {
e.preventDefault();
body = document.getElementsByTagName('body')[0];
bodyClasses = body.classList.toggle('collapse-sidebar');
});
var sections = document.querySelectorAll('aside.sidebar > section');
if (sections.length > 1) {
for (index = 0; index < sections.length; index++) {
section = sections[index];
if ((sections.length >= 3) && index % 3 === 0) {
section.classList.add("first");
}
var count = ((index +1) % 2) ? "odd" : "even";
section.classList.add(count);
}
}
if (sections.length >= 3) {
document.querySelector('aside.sidebar').classList.add('thirds');
}
});
</script>
<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-91189608-1', 'auto');
ga('send', 'pageview');
</script>
</head>
<body>
<header role="banner"><hgroup>
<h1><a href="https://sthbrx.github.io/">Store Halfword Byte-Reverse Indexed</a></h1>
<h2>A Power Technical Blog</h2>
</hgroup></header>
<nav role="navigation"><ul class="subscription" data-subscription="rss">
<li><a href="https://sthbrx.github.io/rss.xml" rel="subscribe-rss">RSS</a></li>
</ul>
<ul class="main-navigation">
<li >
<a href="https://sthbrx.github.io/category/cryptography.html">Cryptography</a>
</li>
<li >
<a href="https://sthbrx.github.io/category/development.html">Development</a>
</li>
<li >
<a href="https://sthbrx.github.io/category/education.html">Education</a>
</li>
<li >
<a href="https://sthbrx.github.io/category/openpower.html">OpenPOWER</a>
</li>
<li >
<a href="https://sthbrx.github.io/category/performance.html">Performance</a>
</li>
<li >
<a href="https://sthbrx.github.io/category/petitboot.html">Petitboot</a>
</li>
<li >
<a href="https://sthbrx.github.io/category/snowpatch.html">snowpatch</a>
</li>
<li >
<a href="https://sthbrx.github.io/category/virtualisation-and-emulation.html">Virtualisation and Emulation</a>
</li>
</ul></nav>
<div id="main">
<div id="content">
<div class="blog-index">
<article>
<header>
<h1 class="entry-title">
<a href="https://sthbrx.github.io/blog/2015/10/30/docker-just-stop-using-aufs/">Docker: Just Stop Using AUFS</a>
</h1>
<p class="meta">
<time datetime="2015-10-30T13:30:00+11:00" pubdate>Fri 30 October 2015</time> </p>
</header>
<div class="byline_index">
<span class="byline author vcard">
Posted by <span class="fn">
<a href="https://sthbrx.github.io/author/daniel-axtens.html">Daniel Axtens</a>
</span>
</span>
<time datetime="2015-10-30T13:30:00+11:00" pubdate>Fri 30 October 2015</time></div>
<div class="entry-content"><p>Docker's default storage driver on most Ubuntu installs is AUFS.</p>
<p>Don't use it. Use Overlay instead. Here's why.</p>
<p>First, some background. I'm testing the performance of the basic LAMP
stack on POWER. (LAMP is Linux + Apache + MySQL/MariaDB + PHP, by the
way.) To do more reliable and repeatable tests, I do my builds and
tests in Docker containers. (See <a href="/blog/2015/10/12/a-tale-of-two-dockers/">my previous post</a> for more info.)</p>
<p>Each test downloads the source of Apache, MariaDB and PHP, and builds
them. This should be quick: the POWER8 system I'm building on has 160
hardware threads and 128 GB of memory. But I was finding that it was
only just keeping pace with a 2 core Intel VM on BlueMix.</p>
<p>Why? Well, my first point of call was to observe a compilation under
<code>top</code>. The header is below.</p>
<p><img alt="top header, showing over 70 percent of CPU time spent in the kernel" src="/images/dja/aufs/top-bad.png"></p>
<p>Over 70% of CPU time is spent in the kernel?! That's weird. Let's dig
deeper.</p>
<p>My next port of call for analysis of CPU-bound workloads is
<code>perf</code>. <code>perf top</code> reports astounding quantities of time in
spin-locks:</p>
<p><img alt="display from perf top, showing 80 percent of time in a spinlock" src="/images/dja/aufs/perf-top-spinlock.png"></p>
<p><code>perf top -g</code> gives us some more information: the time is in system
calls. <code>open()</code> and <code>stat()</code> are the key culprits, and we can see a
number of file system functions are in play in the call-chains of the
spinlocks.</p>
<p><img alt="display from perf top -g, showing syscalls and file ops" src="/images/dja/aufs/perf-top-syscalls.png"></p>
<p>Why are open and stat slow? Well, I know that the files are on an AUFS
mount. (<code>docker info</code> will tell you what you're using if you're not
sure.) So, being something of a kernel hacker, I set out to find out
why. This did not go well. AUFS isn't upstream, it's a separate patch
set. Distros have been trying to deprecate it for years. Indeed, RHEL
doesn't ship it. (To it's credit, Docker seems to be trying to move
away from it.)</p>
<p>Wanting to avoid the minor nightmare that is an out-of-tree patchset,
I looked at other storage drivers for Docker. <a href="https://jpetazzo.github.io/assets/2015-03-03-not-so-deep-dive-into-docker-storage-drivers.html">This presentation is particularly good.</a>
My choices are pretty simple: AUFS, btrfs, device-mapper or
Overlay. Overlay was an obvious choice: it doesn't need me to set up
device mapper on a cloud VM, or reformat things as btrfs.</p>
<p>It's also easy to set up on Ubuntu:</p>
<ul>
<li>
<p>export/save any docker containers you care about.</p>
</li>
<li>
<p>add <code>--storage-driver=overlay</code> option to <code>DOCKER_OPTS</code> in <code>/etc/default/docker</code>, and restart docker (<code>service docker restart</code>)</p>
</li>
<li>
<p>import/load the containters you exported</p>
</li>
<li>
<p>verify that things work, then clear away your old storage directory (<code>/var/lib/docker/aufs</code>). </p>
</li>
</ul>
<p>Having moved my base container across, I set off another build.</p>
<p>The first thing I noticed is that images are much slower to create with Overlay. But once that finishes, and a compile starts, things run much better:</p>
<p><img alt="top, showing close to zero system time, and around 90 percent user time" src="/images/dja/aufs/top-good.png"></p>
<p>The compiles went from taking painfully long to astonishingly fast. Winning.</p>
<p>So in conclusion:</p>
<ul>
<li>
<p>If you use Docker for something that involves open()ing or stat()ing files</p>
</li>
<li>
<p>If you want your machine to do real work, rather than spin in spinlocks</p>
</li>
<li>
<p>If you want to use code that's upstream and thus much better supported</p>
</li>
<li>
<p>If you want something less disruptive than the btrfs or dm storage drivers</p>
</li>
</ul>
<p>...then drop AUFS and switch to Overlay today.</p></div>
</article>
<article>
<header>
<h1 class="entry-title">
<a href="https://sthbrx.github.io/blog/2015/10/12/a-tale-of-two-dockers/">A tale of two Dockers</a>
</h1>
<p class="meta">
<time datetime="2015-10-12T14:14:00+11:00" pubdate>Mon 12 October 2015</time> </p>
</header>
<div class="byline_index">
<span class="byline author vcard">
Posted by <span class="fn">
<a href="https://sthbrx.github.io/author/daniel-axtens.html">Daniel Axtens</a>
</span>
</span>
<time datetime="2015-10-12T14:14:00+11:00" pubdate>Mon 12 October 2015</time></div>
<div class="entry-content"><p>(This was published in an internal technical journal last week, and is now being published here. If you already know what Docker is, feel free to skim the first half.)</p>
<p>Docker seems to be the flavour of the month in IT. Most attention is focussed on using Docker for the deployment of production services. But that's not all Docker is good for. Let's explore Docker, and two ways I use it as a software developer.</p>
<p>Docker: what is it?</p>
<p>Docker is essentially a set of tools to deal with <em>containers</em> and <em>images</em>. </p>
<p>To make up an artificial example, say you are developing a web app. You first build an <em>image</em>: a file system which contains the app, and some associated metadata. The app has to run on something, so you also install things like Python or Ruby and all the necessary libraries, usually by installing a minimal Ubuntu and any necessary packages.<sup id="fnref:1"><a class="footnote-ref" href="#fn:1">1</a></sup> You then run the image inside an isolated environment called a <em>container</em>.</p>
<p>You can have multiple containers running the same image, (for example, your web app running across a fleet of servers) and the containers don't affect each other. Why? Because Docker is designed around the concept of <em>immutability</em>. Containers can write to the image they are running, but the changes are specific to that container, and aren't preserved beyond the life of the container.<sup id="fnref:2"><a class="footnote-ref" href="#fn:2">2</a></sup> Indeed, once built, images can't be changed at all, only rebuilt from scratch.</p>
<p>However, as well as enabling you to easily run multiple copies, another upshot of immutability is that if your web app allows you to upload photos, and you restart the container, your photos will be gone. Your web app needs to be designed to store all of the data outside of the container, sending it to a dedicated database or object store of some sort.</p>
<p>Making your application Docker friendly is significantly more work than just spinning up a virtual machine and installing stuff. So what does all this extra work get you? Three main things: isolation, control and, as mentioned, immutability. </p>
<p><em>Isolation</em> makes containers easy to migrate and deploy, and easy to update. Once an image is built, it can be copied to another system and launched. Isolation also makes it easy to update software your app depends on: you rebuild the image with software updates, and then just deploy it. You don't have to worry about service A relying on version X of a library while service B depends on version Y; it's all self contained. </p>
<p><em>Immutability</em> also helps with upgrades, especially when deploying them across multiple servers. Normally, you would upgrade your app on each server, and have to make sure that every server gets all the same sets of updates. With Docker, you don't upgrade a running container. Instead, you rebuild your Docker image and re-deploy it, and you then know that the same version of everything is running everywhere. This immutability also guards against the situation where you have a number of different servers that are all special snowflakes with their own little tweaks, and you end up with a fractal of complexity.</p>
<p>Finally, Docker offers a lot of <em>control</em> over containers, and for a low performance penalty. Docker containers can have their CPU, memory and network controlled easily, without the overhead of a full virtual machine. This makes it an attractive solution for running untrusted executables.<sup id="fnref:3"><a class="footnote-ref" href="#fn:3">3</a></sup></p>
<p>As an aside: despite the hype, very little of this is actually particularly new. Isolation and control are not new problems. All Unixes, including Linux, support 'chroots'. The name comes from “change root”: the system call changes the processes idea of what the file system root is, making it impossible for it to access things outside of the new designated root directory. FreeBSD has jails, which are more powerful, Solaris has Zones, and AIX has WPARs. Chroots are fast and low overhead. However, they offer much lower ability to control the use of system resources. At the other end of the scale, virtual machines (which have been around since ancient IBM mainframes) offer isolation much better than Docker, but with a greater performance hit.</p>
<p>Similarly, immutability isn't really new: Heroku and AWS Spot Instances are both built around the model that you get resources in a known, consistent state when you start, but in both cases your changes won't persist. In the development world, modern CI systems like Travis CI also have this immutable or disposable model – and this was originally built on VMs. Indeed, with a little bit of extra work, both chroots and VMs can give the same immutability properties that Docker gives.</p>
<p>The control properties that Docker provides are largely as a result of leveraging some Linux kernel concepts, most notably something called namespaces.</p>
<p>What Docker does well is not something novel, but the engineering feat of bringing together fine-grained control, isolation and immutability, and – importantly – a tool-chain that is easier to use than any of the alternatives. Docker's tool-chain eases a lot of pain points with regards to building containers: it's vastly simpler than chroots, and easier to customise than most VM setups. Docker also has a number of engineering tricks to reduce the disk space overhead of isolation.</p>
<p>So, to summarise: Docker provides a toolkit for isolated, immutable, finely controlled containers to run executables and services.</p>
<h2>Docker in development: why?</h2>
<p>I don't run network services at work; I do performance work. So how do I use Docker?</p>
<p>There are two things I do with Docker: I build PHP 5, and do performance regression testing on PHP 7. They're good case studies of how isolation and immutability provide real benefits in development and testing, and how the Docker tool chain makes life a lot nicer that previous solutions.</p>
<h3>PHP 5 builds</h3>
<p>I use the <em>isolation</em> that Docker provides to make building PHP 5 easier. PHP 5 depends on an old version of Bison, version 2. Ubuntu and Debian long since moved to version 3. There are a few ways I could have solved this:</p>
<ul>
<li>I could just install the old version directly on my system in <code>/usr/local/</code>, and hope everything still works and nothing else picks up Bison 2 when it needs Bison 3. Or I could install it somewhere else and remember to change my path correctly before I build PHP 5.</li>
<li>I could roll a chroot by hand. Even with tools like debootstrap and schroot, working in chroots is a painful process.</li>
<li>I could spin up a virtual machine on one of our development boxes and install the old version on that. That feels like overkill: why should I need to run an entire operating system? Why should I need to copy my source tree over the network to build it?</li>
</ul>
<p>Docker makes it easy to have a self-contained environment that has Bison 2 built from source, and to build my latest source tree in that environment. Why is Docker so much easier?</p>
<p>Firstly, Docker allows me to base my container on an existing container, and there's an online library of containers to build from.<sup id="fnref:4"><a class="footnote-ref" href="#fn:4">4</a></sup> This means I don't have to roll a base image with <code>debootstrap</code> or the RHEL/CentOS/Fedora equivalent.</p>
<p>Secondly, unlike a chroot build process, which ultimately is just copying files around, a docker build process includes the ability to both copy files from the host and <em>run commands</em> in the context of the image. This is defined in a file called a <code>Dockerfile</code>, and is kicked off by a single command: <code>docker build</code>.</p>
<p>So, my PHP 5 build container loads an Ubuntu Vivid base container, uses apt-get to install the compiler, tool-chain and headers required to build PHP 5, then installs old bison from source, copies in the PHP source tree, and builds it. The vast majority of this process – the installation of the compiler, headers and bison, can be cached, so they don't have to be downloaded each time. And once the container finishes building, I have a fully built PHP interpreter ready for me to interact with.</p>
<p>I do, at the moment, rebuild PHP 5 from scratch each time. This is a bit sub-optimal from a performance point of view. I could alleviate this with a Docker volume, which is a way of sharing data persistently between a host and a guest, but I haven't been sufficiently bothered by the speed yet. However, Docker volumes are also quite fiddly, leading to the development of tools like <code>docker compose</code> to deal with them. They also are prone to subtle and difficult to debug permission issues.</p>
<h3>PHP 7 performance regression testing</h3>
<p>The second thing I use docker for takes advantage of the throwaway nature of docker environments to prevent cross-contamination.</p>
<p>PHP 7 is the next big version of PHP, slated to be released quite soon. I care about how that runs on POWER, and I preferably want to know if it suddenly deteriorates (or improves!). I use Docker to build a container with a daily build of PHP 7, and then I run a benchmark in it. This doesn't give me a particularly meaningful absolute number, but it allows me to track progress over time. Building it inside of Docker means that I can be sure that nothing from old runs persists into new runs, thus giving me more reliable data. However, because I do want the timing data I collect to persist, I send it out of the container over the network.</p>
<p>I've now been collecting this data for almost 4 months, and it's plotted below, along with a 5-point moving average. The most notable feature of the graph is a the drop in benchmark time at about the middle. Sure enough, if you look at the PHP repository, you will see that a set of changes to improve PHP performance were merged on July 29: changes submitted by our very own Anton Blanchard.<sup id="fnref:5"><a class="footnote-ref" href="#fn:5">5</a></sup></p>
<p><img alt="Graph of PHP 7 performance over time" src="/images/dja/php7-perf.png"></p>
<h2>Docker pain points</h2>
<p>Docker provides a vastly improved experience over previous solutions, but there are still a few pain points. For example:</p>
<ol>
<li>
<p>Docker was apparently written by people who had no concept that platforms other than x86 exist. This leads to major issues for cross-architectural setups. For instance, Docker identifies images by a name and a revision. For example, <code>ubuntu</code> is the name of an image, and <code>15.04</code> is a revision. There's no ability to specify an architecture. So, how you do specify that you want, say, a 64-bit, little-endian PowerPC build of an image versus an x86 build? There have been a couple of approaches, both of which are pretty bad. You could name the image differently: say <code>ubuntu_ppc64le</code>. You can also just cheat and override the <code>ubuntu</code> name with an architecture specific version. Both of these break some assumptions in the Docker ecosystem and are a pain to work with.</p>
</li>
<li>
<p>Image building is incredibly inflexible. If you have one system that requires a proxy, and one that does not, you need different Dockerfiles. As far as I can tell, there are no simple ways to hook in any changes between systems into a generic Dockerfile. This is largely by design, but it's still really annoying when you have one system behind a firewall and one system out on the public cloud (as I do in the PHP 7 setup).</p>
</li>
<li>
<p>Visibility into a Docker server is poor. You end up with lots of different, anonymous images and dead containers, and you end up needing scripts to clean them up. It's not clear what Docker puts on your file system, or where, or how to interact with it.</p>
</li>
<li>
<p>Docker is still using reasonably new technologies. This leads to occasional weird, obscure and difficult to debug issues.<sup id="fnref:6"><a class="footnote-ref" href="#fn:6">6</a></sup></p>
</li>
</ol>
<h2>Final words</h2>
<p>Docker provides me with a lot of useful tools in software development: both in terms of building and testing. Making use of it requires a certain amount of careful design thought, but when applied thoughtfully it can make life significantly easier.</p>
<div class="footnote">
<hr>
<ol>
<li id="fn:1">
<p>There's some debate about how much stuff from the OS installation you should be using. You need to have key dynamic libraries available, but I would argue that you shouldn't be running long running processes other than your application. You shouldn't, for example, be running a SSH daemon in your container. (The one exception is that you must handle orphaned child processes appropriately: see <a href="https://blog.phusion.nl/2015/01/20/docker-and-the-pid-1-zombie-reaping-problem/">https://blog.phusion.nl/2015/01/20/docker-and-the-pid-1-zombie-reaping-problem/</a>) Considerations like debugging and monitoring the health of docker containers mean that this point of view is not universally shared. <a class="footnote-backref" href="#fnref:1" title="Jump back to footnote 1 in the text">↩</a></p>
</li>
<li id="fn:2">
<p>Why not simply make them read only? You may be surprised at how many things break when running on a read-only file system. Things like logs and temporary files are common issues. <a class="footnote-backref" href="#fnref:2" title="Jump back to footnote 2 in the text">↩</a></p>
</li>
<li id="fn:3">
<p>It is, however, easier to escape a Docker container than a VM. In Docker, an untrusted executable only needs a kernel exploit to get to root on the host, whereas in a VM you need a guest-to-host vulnerability, which are much rarer. <a class="footnote-backref" href="#fnref:3" title="Jump back to footnote 3 in the text">↩</a></p>
</li>
<li id="fn:4">
<p>Anyone can upload an image, so this does require running untrusted code from the Internet. Sadly, this is a distinctly retrograde step when compared to the process of installing binary packages in distros, which are all signed by a distro's private key. <a class="footnote-backref" href="#fnref:4" title="Jump back to footnote 4 in the text">↩</a></p>
</li>
<li id="fn:5">
<p>See <a href="https://github.com/php/php-src/pull/1326">https://github.com/php/php-src/pull/1326</a> <a class="footnote-backref" href="#fnref:5" title="Jump back to footnote 5 in the text">↩</a></p>
</li>
<li id="fn:6">
<p>I hit this last week: <a href="https://github.com/docker/docker/issues/16256">https://github.com/docker/docker/issues/16256</a>, although maybe that's my fault for running systemd on my laptop. <a class="footnote-backref" href="#fnref:6" title="Jump back to footnote 6 in the text">↩</a></p>
</li>
</ol>
</div></div>
</article>
<article>
<header>
<h1 class="entry-title">
<a href="https://sthbrx.github.io/blog/2015/06/03/ppc64le-hello-on-real-hardware/">Running ppc64le_hello on real hardware</a>
</h1>
<p class="meta">
<time datetime="2015-06-03T12:16:00+10:00" pubdate>Wed 03 June 2015</time> </p>
</header>
<div class="byline_index">
<span class="byline author vcard">
Posted by <span class="fn">
<a href="https://sthbrx.github.io/author/daniel-axtens.html">Daniel Axtens</a>
</span>
</span>
<time datetime="2015-06-03T12:16:00+10:00" pubdate>Wed 03 June 2015</time></div>
<div class="entry-content"><p>So today I saw <a href="https://github.com/andreiw/ppc64le_hello">Freestanding “Hello World” for OpenPower</a> on <a href="https://news.ycombinator.com/item?id=9649490">Hacker News</a>. Sadly Andrei hadn't been able to test it on real hardware, so I set out to get it running on a real OpenPOWER box. Here's what I did.</p>
<p>Firstly, clone the repo, and, as mentioned in the README, comment out <code>mambo_write</code>. Build it.</p>
<p>Grab <a href="https://github.com/open-power/op-build">op-build</a>, and build a Habanero defconfig. To save yourself a fair bit of time, first edit <code>openpower/configs/habanero_defconfig</code> to answer <code>n</code> about a custom kernel source. That'll save you hours of waiting for git.</p>
<p>This will build you a PNOR that will boot a linux kernel with Petitboot. This is almost what you want: you need Skiboot, Hostboot and a bunch of the POWER specific bits and bobs, but you don't actually want the Linux boot kernel.</p>
<p>Then, based on <code>op-build/openpower/package/openpower-pnor/openpower-pnor.mk</code>, we look through the output of <code>op-build</code> for a <code>create_pnor_image.pl</code> command, something like this monstrosity:</p>
<p><code>PATH="/scratch/dja/public/op-build/output/host/bin:/scratch/dja/public/op-build/output/host/sbin:/scratch/dja/public/op-build/output/host/usr/bin:/scratch/dja/public/op-build/output/host/usr/sbin:/home/dja/bin:/home/dja/bin:/home/dja/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games:/opt/openpower/common/x86_64/bin" /scratch/dja/public/op-build/output/build/openpower-pnor-ed1682e10526ebd85825427fbf397361bb0e34aa/create_pnor_image.pl -xml_layout_file /scratch/dja/public/op-build/output/build/openpower-pnor-ed1682e10526ebd85825427fbf397361bb0e34aa/"defaultPnorLayoutWithGoldenSide.xml" -pnor_filename /scratch/dja/public/op-build/output/host/usr/powerpc64-buildroot-linux-gnu/sysroot/pnor/"habanero.pnor" -hb_image_dir /scratch/dja/public/op-build/output/host/usr/powerpc64-buildroot-linux-gnu/sysroot/hostboot_build_images/ -scratch_dir /scratch/dja/public/op-build/output/host/usr/powerpc64-buildroot-linux-gnu/sysroot/openpower_pnor_scratch/ -outdir /scratch/dja/public/op-build/output/host/usr/powerpc64-buildroot-linux-gnu/sysroot/pnor/ -payload /scratch/dja/public/op-build/output/images/"skiboot.lid" -bootkernel /scratch/dja/public/op-build/output/images/zImage.epapr -sbe_binary_filename "venice_sbe.img.ecc" -sbec_binary_filename "centaur_sbec_pad.img.ecc" -wink_binary_filename "p8.ref_image.hdr.bin.ecc" -occ_binary_filename /scratch/dja/public/op-build/output/host/usr/powerpc64-buildroot-linux-gnu/sysroot/occ/"occ.bin" -targeting_binary_filename "HABANERO_HB.targeting.bin.ecc" -openpower_version_filename /scratch/dja/public/op-build/output/host/usr/powerpc64-buildroot-linux-gnu/sysroot/openpower_version/openpower-pnor.version.txt</code></p>
<p>Replace the <code>-bootkernel</code> arguement with the path to ppc64le_hello, e.g.: <code>-bootkernel /scratch/dja/public/ppc64le_hello/ppc64le_hello</code></p>
<p>Don't forget to move it into place! </p>
<div class="highlight"><pre><span></span><code>mv<span class="w"> </span>output/host/usr/powerpc64-buildroot-linux-gnu/sysroot/pnor/habanero.pnor<span class="w"> </span>output/images/habanero.pnor
</code></pre></div>
<p>Then we can use skiboot's boot test script (written by Cyril and me, coincidentally!) to flash it.</p>
<div class="highlight"><pre><span></span><code>ppc64le_hello/skiboot/external/boot-tests/boot_test.sh<span class="w"> </span>-vp<span class="w"> </span>-t<span class="w"> </span>hab2-bmc<span class="w"> </span>-P<span class="w"> </span><path<span class="w"> </span>to>/habanero.pnor
</code></pre></div>
<p>It's not going to get into Petitboot, so just interrupt it after it powers up the box and connect with IPMI. It boots, kinda:</p>
<div class="highlight"><pre><span></span><code>[11012941323,5] INIT: Starting kernel at 0x20010000, fdt at 0x3044db68 (size 0x11cc3)
Hello OPAL!
_start = 0x20010000
_bss = 0x20017E28
_stack = 0x20018000
_end = 0x2001A000
KPCR = 0x20017E50
OPAL = 0x30000000
FDT = 0x3044DB68
CPU0 not found?
Pick your poison:
Choices: (MMU = disabled):
(d) 5s delay
(e) test exception
(n) test nested exception
(f) dump FDT
(M) enable MMU
(m) disable MMU
(t) test MMU
(u) test non-priviledged code
(I) enable ints
(i) disable ints
(H) enable HV dec
(h) disable HV dec
(q) poweroff
1.42486|ERRL|Dumping errors reported prior to registration
</code></pre></div>
<p>Yes, it does wrap horribly. However, the big issue here (which you'll have to scroll to see!) is the "CPU0 not found?". Fortunately, we can fix this with a little patch to <code>cpu_init</code> in main.c to test for a PowerPC POWER8:</p>
<div class="highlight"><pre><span></span><code><span class="w"> </span><span class="n">cpu0_node</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">fdt_path_offset</span><span class="p">(</span><span class="n">fdt</span><span class="p">,</span><span class="w"> </span><span class="s">"/cpus/cpu@0"</span><span class="p">);</span>
<span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">cpu0_node</span><span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="mi">0</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="n">cpu0_node</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">fdt_path_offset</span><span class="p">(</span><span class="n">fdt</span><span class="p">,</span><span class="w"> </span><span class="s">"/cpus/PowerPC,POWER8@20"</span><span class="p">);</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">cpu0_node</span><span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="mi">0</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="n">printk</span><span class="p">(</span><span class="s">"CPU0 not found?</span><span class="se">\n</span><span class="s">"</span><span class="p">);</span>
<span class="w"> </span><span class="k">return</span><span class="p">;</span>
<span class="w"> </span><span class="p">}</span>
</code></pre></div>
<p>This is definitely the <em>wrong</em> way to do this, but it works for now.</p>
<p>Now, correcting for weird wrapping, we get:</p>
<div class="highlight"><pre><span></span><code>Hello OPAL!
_start = 0x20010000
_bss = 0x20017E28
_stack = 0x20018000
_end = 0x2001A000
KPCR = 0x20017E50
OPAL = 0x30000000
FDT = 0x3044DB68
Assuming default SLB size
SLB size = 0x20
TB freq = 512000000
[13205442015,3] OPAL: Trying a CPU re-init with flags: 0x2
Unrecoverable exception stack top @ 0x20019EC8
HTAB (2048 ptegs, mask 0x7FF, size 0x40000) @ 0x20040000
SLB entries:
1: E 0x8000000 V 0x4000000000000400
EA 0x20040000 -> hash 0x20040 -> pteg 0x200 = RA 0x20040000
EA 0x20041000 -> hash 0x20041 -> pteg 0x208 = RA 0x20041000
EA 0x20042000 -> hash 0x20042 -> pteg 0x210 = RA 0x20042000
EA 0x20043000 -> hash 0x20043 -> pteg 0x218 = RA 0x20043000
EA 0x20044000 -> hash 0x20044 -> pteg 0x220 = RA 0x20044000
EA 0x20045000 -> hash 0x20045 -> pteg 0x228 = RA 0x20045000
EA 0x20046000 -> hash 0x20046 -> pteg 0x230 = RA 0x20046000
EA 0x20047000 -> hash 0x20047 -> pteg 0x238 = RA 0x20047000
EA 0x20048000 -> hash 0x20048 -> pteg 0x240 = RA 0x20048000
...
</code></pre></div>
<p>The weird wrapping seems to be caused by NULLs getting printed to OPAL, but I haven't traced what causes that.</p>
<p>Anyway, now it largely works! Here's a transcript of some things it can do on real hardware.</p>
<div class="highlight"><pre><span></span><code>Choices: (MMU = disabled):
(d) 5s delay
(e) test exception
(n) test nested exception
(f) dump FDT
(M) enable MMU
(m) disable MMU
(t) test MMU
(u) test non-priviledged code
(I) enable ints
(i) disable ints
(H) enable HV dec
(h) disable HV dec
(q) poweroff
<press e>
Testing exception handling...
sc(feed) => 0xFEEDFACE
Choices: (MMU = disabled):
(d) 5s delay
(e) test exception
(n) test nested exception
(f) dump FDT
(M) enable MMU
(m) disable MMU
(t) test MMU
(u) test non-priviledged code
(I) enable ints
(i) disable ints
(H) enable HV dec
(h) disable HV dec
(q) poweroff
<press t>
EA 0xFFFFFFF000 -> hash 0xFFFFFFF -> pteg 0x3FF8 = RA 0x20010000
mapped 0xFFFFFFF000 to 0x20010000 correctly
EA 0xFFFFFFF000 -> hash 0xFFFFFFF -> pteg 0x3FF8 = unmap
EA 0xFFFFFFF000 -> hash 0xFFFFFFF -> pteg 0x3FF8 = RA 0x20011000
mapped 0xFFFFFFF000 to 0x20011000 incorrectly
EA 0xFFFFFFF000 -> hash 0xFFFFFFF -> pteg 0x3FF8 = unmap
Choices: (MMU = disabled):
(d) 5s delay
(e) test exception
(n) test nested exception
(f) dump FDT
(M) enable MMU
(m) disable MMU
(t) test MMU
(u) test non-priviledged code
(I) enable ints
(i) disable ints
(H) enable HV dec
(h) disable HV dec
(q) poweroff
<press u>
EA 0xFFFFFFF000 -> hash 0xFFFFFFF -> pteg 0x3FF8 = RA 0x20080000
returning to user code
returning to kernel code
EA 0xFFFFFFF000 -> hash 0xFFFFFFF -> pteg 0x3FF8 = unmap
</code></pre></div>
<p>I also tested the other functions and they all seem to work. Running non-priviledged code with the MMU on works. Dumping the FDT and the 5s delay both worked, although they tend to stress IPMI a <em>lot</em>. The delay seems to correspond well with real time as well.</p>
<p>It does tend to error out and reboot quite often, usually on the menu screen, for reasons that are not clear to me. It usually starts with something entirely uninformative from Hostboot, like this:</p>
<div class="highlight"><pre><span></span><code>1.41801|ERRL|Dumping errors reported prior to registration
2.89873|Ignoring boot flags, incorrect version 0x0
</code></pre></div>
<p>That may be easy to fix, but again I haven't had time to trace it.</p>
<p>All in all, it's very exciting to see something come out of the simulator and in to real hardware. Hopefully with the proliferation of OpenPOWER hardware, prices will fall and these sorts of systems will become increasingly accessible to people with cool low level projects like this!</p></div>
</article>
<article>
<header>
<h1 class="entry-title">
<a href="https://sthbrx.github.io/blog/2015/06/02/autoboot/">Petitboot Autoboot Changes</a>
</h1>
<p class="meta">
<time datetime="2015-06-02T08:11:00+10:00" pubdate>Tue 02 June 2015</time> </p>
</header>
<div class="byline_index">
<span class="byline author vcard">
Posted by <span class="fn">
<a href="https://sthbrx.github.io/author/samuel-mendoza-jonas.html">Samuel Mendoza-Jonas</a>
</span>
</span>
<time datetime="2015-06-02T08:11:00+10:00" pubdate>Tue 02 June 2015</time></div>
<div class="entry-content"><p>The way autoboot behaves in Petitboot has undergone some significant changes recently, so in order to ward off any angry emails lets take a quick tour of how the new system works.</p>
<h2>Old & Busted</h2>
<p>For some context, here is the old (or current depending on what you're running) section of the configuration screen.</p>
<p><img alt="Old Autoboot" src="/images/sammj/oldstyle.jpg"></p>
<p>This gives you three main options: don't autoboot, autoboot from anything, or autoboot only from a specific device. For the majority of installations this is fine, such as when you have only one default option, or know exactly which device you'll be booting from.</p>
<p>A side note about default options: it is important to note that not all boot options are valid <em>autoboot</em> options. A boot option is only considered for auto-booting if it is marked default, eg. 'set default' in GRUB and 'default' in PXE options.</p>
<h2>New Hotness</h2>
<p>Below is the new autoboot configuration.</p>
<p><img alt="New Autoboot" src="/images/sammj/newstyle.jpg"></p>
<p>The new design allows you to specify an ordered list of autoboot options.
The last two of the three buttons are self explanatory - clear the list and autoboot any device, or clear the list completely (no autoboot).</p>
<p>Selecting the first button, 'Add Device' brings up the following screen:</p>
<p><img alt="Device Selection" src="/images/sammj/devices.jpg"></p>
<p>From here you can select any device or <em>class</em> of device to add to the boot order. Once added to the boot order, the order of boot options can be changed with the left and right arrow keys, and removed from the list with the minus key ('-').</p>
<p>This allows you to create additional autoboot configurations such as "Try to boot from sda2, otherwise boot from the network", or "Give priority to PXE options from eth0, otherwise try any other netboot option".
You can retain the original behaviour by only putting one option into the list (either 'Any Device' or a specific device).</p>
<p>Presently you can add any option into the list and order them how you like - which means you can do silly things like this:</p>
<p><img alt="If you send me a bug report with this in it I may laugh at you" src="/images/sammj/redundant.jpg"></p>
<h2>IPMI</h2>
<p>Slightly prior to the boot order changes Petitboot also received an update to its IPMI handling. IPMI 'bootdev' commands allow you to override the current autoboot configuration remotely, either by specifying a device type to boot (eg. PXE), or by forcing Petitboot to boot into the 'setup' or 'safe' modes. IPMI overrides are either persistent or non-persistent. A non-persistent override will disappear after a successful boot - that is, a successful boot of a boot option, not booting to Petitboot itself - whereas a persistent override will, well, persist!</p>
<p>If there is an IPMI override currently active, it will appear in the configuration screen with an option to manually clear it:</p>
<p><img alt="IPMI Overrides" src="/images/sammj/ipmi.jpg"></p>
<hr>
<p>That sums up the recent changes to autoboot; a bit more flexibility in assigning priority, and options for more detailed autoboot order if you need it. New versions of Petitboot are backwards compatible and will recognise older saved settings, so updating your firmware won't cause your machines to start booting things at random.</p></div>
</article>
<article>
<header>
<h1 class="entry-title">
<a href="https://sthbrx.github.io/blog/2015/05/27/joining-the-capi-project/">Joining the CAPI project</a>
</h1>
<p class="meta">
<time datetime="2015-05-27T15:08:00+10:00" pubdate>Wed 27 May 2015</time> </p>
</header>
<div class="byline_index">
<span class="byline author vcard">
Posted by <span class="fn">
<a href="https://sthbrx.github.io/author/daniel-axtens.html">Daniel Axtens</a>
</span>
</span>
<time datetime="2015-05-27T15:08:00+10:00" pubdate>Wed 27 May 2015</time></div>
<div class="entry-content"><p>(I wrote this blog post a couple of months ago, but it's still quite relevant.)</p>
<p>Hi, I'm Daniel! I work in OzLabs, part of IBM's Australian Development Labs. Recently, I've been assigned to the CAPI project, and I've been given the opportunity to give you an idea of what this is, and what I'll be up to in the future!</p>
<h2>What even is CAPI?</h2>
<p>To help you understand CAPI, think back to the time before computers. We had a variety of machines: machines to build things, to check things, to count things, but they were all specialised --- good at one and only one thing.</p>
<p>Specialised machines, while great at their intended task, are really expensive to develop. Not only that, it's often impossible to change how they operate, even in very small ways.</p>
<p>Computer processors, on the other hand, are generalists. They are cheap. They can do a lot of things. If you can break a task down into simple steps, it's easy to get them to do it. The trade-off is that computer processors are incredibly inefficient at everything.</p>
<p>Now imagine, if you will, that a specialised machine is a highly trained and experienced professional, a computer processor is a hungover university student.</p>
<p>Over the years, we've tried lots of things to make student faster. Firstly, we gave the student lots of caffeine to make them go as fast as they can. That worked for a while, but you can only give someone so much caffeine before they become unreliable. Then we tried teaming the student up with another student, so they can do two things at once. That worked, so we added more and more students. Unfortunately, lots of tasks can only be done by one person at a time, and team-work is complicated to co-ordinate. We've also recently noticed that some tasks come up often, so we've given them some tools for those specific tasks. Sadly, the tools are only useful for those specific situations.</p>
<p>Sometimes, what you really need is a professional.</p>
<p>However, there are a few difficulties in getting a professional to work with uni students. They don't speak the same way; they don't think the same way, and they don't work the same way. You need to teach the uni students how to work with the professional, and vice versa.</p>
<p>Previously, developing this interface – this connection between a generalist processor and a specialist machine – has been particularly difficult. The interface between processors and these specialised machines – known as <em>accelerators</em> – has also tended to suffer from bottlenecks and inefficiencies.</p>
<p>This is the problem CAPI solves. CAPI provides a simpler and more optimised way to interface specialised hardware accelerators with IBM's most recent line of processors, POWER8. It's a common 'language' that the processor and the accelerator talk, that makes it much easier to build the hardware side and easier to program the software side. In our Canberra lab, we're working primarily on the operating system side of this. We are working with some external companies who are building CAPI devices and the optimised software products which use them.</p>
<p>From a technical point of view, CAPI provides <em>coherent</em> access to system memory and processor caches, eliminating a major bottleneck in using external devices as accelerators. This is illustrated really well by the following graphic from <a href="https://www.youtube.com/watch?v=4ZyXc12J6FA">an IBM promotional video</a>. In the non-CAPI case, you can see there's a lot of data (the little boxes) stalled in the PCIe subsystem, whereas with CAPI, the accelerator has direct access to the memory subsystem, which makes everything go faster.</p>
<p><img alt="Slide showing CAPI's memory access" src="/images/dja/capi-memory.png"></p>
<h2>Uses of CAPI</h2>
<p>CAPI technology is already powering a few really cool products.</p>
<p>Firstly, we have an implementation of Redis that sits on top of flash storage connected over CAPI. Or, to take out the buzzwords, CAPI lets us do really, really fast NoSQL databases. There's <a href="https://www.youtube.com/watch?v=cCmFc_0xsvA">a video online</a> giving more details.</p>
<p>Secondly, our partner <a href="http://www.mellanox.com/page/products_dyn?product_family=201&mtag=connectx_4_vpi_card">Mellanox</a> is using CAPI to make network cards that run at speeds of up to 100Gb/s.</p>
<p>CAPI is also part of IBM's OpenPOWER initiative, where we're trying to grow a community of companies around our POWER system designs. So in many ways, CAPI is both a really cool technology, and a brand new ecosystem that we're growing here in the Canberra labs. It's very cool to be a part of!</p></div>
</article>
<div class="pagination">
<a class="prev" href="https://sthbrx.github.io/index13.html">← Older</a>
<a class="next" href="https://sthbrx.github.io/index11.html">Newer →</a>
<br />
</div></div>
<aside class="sidebar">
<section>
<h1>Recent Posts</h1>
<ul id="recent_posts">
<li class="post">
<a href="https://sthbrx.github.io/blog/2023/08/07/going-out-on-a-limb-efficient-elliptic-curve-arithmetic-in-openssl/">Going out on a Limb: Efficient Elliptic Curve Arithmetic in OpenSSL</a>
</li>
<li class="post">
<a href="https://sthbrx.github.io/blog/2023/08/04/quirks-of-parsing-ssh-configs/">Quirks of parsing SSH configs</a>
</li>
<li class="post">
<a href="https://sthbrx.github.io/blog/2023/04/05/detecting-rootless-docker/">Detecting rootless Docker</a>
</li>
<li class="post">
<a href="https://sthbrx.github.io/blog/2023/04/04/dumb-bugs-the-pci-device-that-wasnt/">Dumb bugs: the PCI device that wasn't</a>
</li>
<li class="post">
<a href="https://sthbrx.github.io/blog/2023/03/24/dumb-bugs-when-a-date-breaks-booting-the-kernel/">Dumb bugs: When a date breaks booting the kernel</a>
</li>
</ul>
</section>
<section>
<h1>Categories</h1>
<ul id="recent_posts">
<li><a href="https://sthbrx.github.io/category/cryptography.html">Cryptography</a></li>
<li><a href="https://sthbrx.github.io/category/development.html">Development</a></li>
<li><a href="https://sthbrx.github.io/category/education.html">Education</a></li>
<li><a href="https://sthbrx.github.io/category/openpower.html">OpenPOWER</a></li>
<li><a href="https://sthbrx.github.io/category/performance.html">Performance</a></li>
<li><a href="https://sthbrx.github.io/category/petitboot.html">Petitboot</a></li>
<li><a href="https://sthbrx.github.io/category/snowpatch.html">snowpatch</a></li>
<li><a href="https://sthbrx.github.io/category/virtualisation-and-emulation.html">Virtualisation and Emulation</a></li>
</ul>
</section>
<section>
<h1>Tags</h1>
<a href="https://sthbrx.github.io/tag/ssh.html">ssh</a>, <a href="https://sthbrx.github.io/tag/docker.html">Docker</a>, <a href="https://sthbrx.github.io/tag/syzkaller.html">syzkaller</a>, <a href="https://sthbrx.github.io/tag/linux.html">linux</a>, <a href="https://sthbrx.github.io/tag/power8.html">power8</a>, <a href="https://sthbrx.github.io/tag/distro.html">distro</a>, <a href="https://sthbrx.github.io/tag/kernel.html">kernel</a>, <a href="https://sthbrx.github.io/tag/hardening.html">hardening</a>, <a href="https://sthbrx.github.io/tag/testing.html">testing</a>, <a href="https://sthbrx.github.io/tag/conferences.html">conferences</a>, <a href="https://sthbrx.github.io/tag/instruction-set-architecture.html">Instruction Set Architecture</a>, <a href="https://sthbrx.github.io/tag/openpower.html">openpower</a>, <a href="https://sthbrx.github.io/tag/firmware.html">firmware</a>, <a href="https://sthbrx.github.io/tag/goodposts.html">goodposts</a>, <a href="https://sthbrx.github.io/tag/realcontent.html">realcontent</a>, <a href="https://sthbrx.github.io/tag/madposting.html">madposting</a>, <a href="https://sthbrx.github.io/tag/op-test.html">op-test</a>, <a href="https://sthbrx.github.io/tag/qemu.html">qemu</a>, <a href="https://sthbrx.github.io/tag/pci.html">pci</a>, <a href="https://sthbrx.github.io/tag/sparseposting.html">sparseposting</a>, <a href="https://sthbrx.github.io/tag/petitboot.html">petitboot</a>, <a href="https://sthbrx.github.io/tag/security.html">security</a>, <a href="https://sthbrx.github.io/tag/vscode.html">vscode</a>, <a href="https://sthbrx.github.io/tag/code.html">code</a>, <a href="https://sthbrx.github.io/tag/openbmc.html">openbmc</a>, <a href="https://sthbrx.github.io/tag/ipmi.html">ipmi</a>, <a href="https://sthbrx.github.io/tag/opencapi.html">opencapi</a>, <a href="https://sthbrx.github.io/tag/openpower-summit.html">openpower summit</a>, <a href="https://sthbrx.github.io/tag/easyposts.html">easyposts</a>, <a href="https://sthbrx.github.io/tag/linuxboot.html">linuxboot</a>, <a href="https://sthbrx.github.io/tag/google.html">google</a>, <a href="https://sthbrx.github.io/tag/intel.html">intel</a>, <a href="https://sthbrx.github.io/tag/osfc.html">osfc</a>, <a href="https://sthbrx.github.io/tag/shortposts.html">shortposts</a>, <a href="https://sthbrx.github.io/tag/facebook.html">facebook</a>, <a href="https://sthbrx.github.io/tag/performance.html">performance</a>, <a href="https://sthbrx.github.io/tag/phoronix.html">phoronix</a>, <a href="https://sthbrx.github.io/tag/benchmarks.html">benchmarks</a>, <a href="https://sthbrx.github.io/tag/stupid-ideas.html">stupid ideas</a>, <a href="https://sthbrx.github.io/tag/network.html">network</a>, <a href="https://sthbrx.github.io/tag/power.html">power</a>, <a href="https://sthbrx.github.io/tag/xdp.html">xdp</a>, <a href="https://sthbrx.github.io/tag/networking.html">networking</a>, <a href="https://sthbrx.github.io/tag/remoteposts.html">remoteposts</a>, <a href="https://sthbrx.github.io/tag/ceph.html">ceph</a>, <a href="https://sthbrx.github.io/tag/raid.html">raid</a>, <a href="https://sthbrx.github.io/tag/storage.html">storage</a>, <a href="https://sthbrx.github.io/tag/erasure.html">erasure</a>, <a href="https://sthbrx.github.io/tag/lustre.html">lustre</a>, <a href="https://sthbrx.github.io/tag/hpc.html">hpc</a>, <a href="https://sthbrx.github.io/tag/nvlink.html">nvlink</a>, <a href="https://sthbrx.github.io/tag/namd.html">namd</a>, <a href="https://sthbrx.github.io/tag/cuda.html">cuda</a>, <a href="https://sthbrx.github.io/tag/gpu.html">gpu</a>, <a href="https://sthbrx.github.io/tag/minsky.html">minsky</a>, <a href="https://sthbrx.github.io/tag/s822lc-for-hpc.html">S822LC for hpc</a>, <a href="https://sthbrx.github.io/tag/debug.html">debug</a>, <a href="https://sthbrx.github.io/tag/virtualisation.html">virtualisation</a>, <a href="https://sthbrx.github.io/tag/dmesg.html">dmesg</a>, <a href="https://sthbrx.github.io/tag/printk.html">printk</a>, <a href="https://sthbrx.github.io/tag/boot.html">boot</a>, <a href="https://sthbrx.github.io/tag/early.html">early</a>, <a href="https://sthbrx.github.io/tag/error.html">error</a>, <a href="https://sthbrx.github.io/tag/centos.html">centos</a>, <a href="https://sthbrx.github.io/tag/centos7.html">centos7</a>, <a href="https://sthbrx.github.io/tag/p8.html">p8</a>, <a href="https://sthbrx.github.io/tag/bmc.html">bmc</a>, <a href="https://sthbrx.github.io/tag/rhel.html">RHEL</a>, <a href="https://sthbrx.github.io/tag/skiroot.html">skiroot</a>, <a href="https://sthbrx.github.io/tag/devmapper.html">devmapper</a>, <a href="https://sthbrx.github.io/tag/lvm.html">lvm</a>, <a href="https://sthbrx.github.io/tag/cgroups.html">cgroups</a>, <a href="https://sthbrx.github.io/tag/numa.html">numa</a>, <a href="https://sthbrx.github.io/tag/development.html">Development</a>, <a href="https://sthbrx.github.io/tag/netboot.html">netboot</a>, <a href="https://sthbrx.github.io/tag/pxe.html">pxe</a>, <a href="https://sthbrx.github.io/tag/education.html">Education</a>, <a href="https://sthbrx.github.io/tag/work-experience.html">work experience</a>, <a href="https://sthbrx.github.io/tag/asm.html">asm</a>, <a href="https://sthbrx.github.io/tag/vdso.html">vdso</a>, <a href="https://sthbrx.github.io/tag/snowpatch.html">snowpatch</a>, <a href="https://sthbrx.github.io/tag/tools.html">tools</a>, <a href="https://sthbrx.github.io/tag/intern.html">intern</a>, <a href="https://sthbrx.github.io/tag/srop.html">SROP</a>, <a href="https://sthbrx.github.io/tag/mitigation.html">mitigation</a>, <a href="https://sthbrx.github.io/tag/double.html">double</a>, <a href="https://sthbrx.github.io/tag/float.html">float</a>, <a href="https://sthbrx.github.io/tag/hex.html">hex</a>, <a href="https://sthbrx.github.io/tag/debugging.html">debugging</a>, <a href="https://sthbrx.github.io/tag/skiboot.html">skiboot</a>, <a href="https://sthbrx.github.io/tag/opal.html">OPAL</a>, <a href="https://sthbrx.github.io/tag/fsp.html">FSP</a>, <a href="https://sthbrx.github.io/tag/patches.html">patches</a>, <a href="https://sthbrx.github.io/tag/based16.html">based16</a>, <a href="https://sthbrx.github.io/tag/linux-gods.html">Linux Gods</a>, <a href="https://sthbrx.github.io/tag/ozlabs.html">Ozlabs</a>, <a href="https://sthbrx.github.io/tag/offtopic.html">offtopic</a>, <a href="https://sthbrx.github.io/tag/autoboot.html">autoboot</a>, <a href="https://sthbrx.github.io/tag/kexec.html">kexec</a>, <a href="https://sthbrx.github.io/tag/aufs.html">aufs</a>, <a href="https://sthbrx.github.io/tag/overlay.html">overlay</a>, <a href="https://sthbrx.github.io/tag/php.html">php</a>, <a href="https://sthbrx.github.io/tag/capi.html">capi</a> </section>
<section>
<h1><a href="https://sthbrx.github.io/authors.html">Authors</a></h1>
<ul id="authors_list">
<li><a href="https://sthbrx.github.io/author/alastair-dsilva.html">Alastair D'Silva</a></li>
<li><a href="https://sthbrx.github.io/author/andrew-donnellan.html">Andrew Donnellan</a></li>
<li><a href="https://sthbrx.github.io/author/anton-blanchard.html">Anton Blanchard</a></li>
<li><a href="https://sthbrx.github.io/author/benjamin-gray.html">Benjamin Gray</a></li>
<li><a href="https://sthbrx.github.io/author/callum-scarvell.html">Callum Scarvell</a></li>
<li><a href="https://sthbrx.github.io/author/cyril-bur.html">Cyril Bur</a></li>
<li><a href="https://sthbrx.github.io/author/daniel-axtens.html">Daniel Axtens</a></li>
<li><a href="https://sthbrx.github.io/author/daniel-black.html">Daniel Black</a></li>
<li><a href="https://sthbrx.github.io/author/joel-stanley.html">Joel Stanley</a></li>
<li><a href="https://sthbrx.github.io/author/nick-piggin.html">Nick Piggin</a></li>
<li><a href="https://sthbrx.github.io/author/rashmica-gupta.html">Rashmica Gupta</a></li>
<li><a href="https://sthbrx.github.io/author/rohan-mclure.html">Rohan McLure</a></li>
<li><a href="https://sthbrx.github.io/author/russell-currey.html">Russell Currey</a></li>
<li><a href="https://sthbrx.github.io/author/samuel-mendoza-jonas.html">Samuel Mendoza-Jonas</a></li>
<li><a href="https://sthbrx.github.io/author/suraj-jitindar-singh.html">Suraj Jitindar Singh</a></li>
</ul>
</section>
<section>
<h1>Social</h1>
<ul>
<li><a href="https://sthbrx.github.io/rss.xml" type="application/rss+xml" rel="alternate">RSS</a></li>
<li><a href="https://github.com/sthbrx/" target="_blank">GitHub</a></li>
<li><a href="https://lists.ozlabs.org/listinfo/linuxppc-dev" target="_blank">linuxppc mailing list</a></li>
<li><a href="https://lists.ozlabs.org/listinfo/skiboot" target="_blank">Skiboot mailing list</a></li>
</ul>
</section>
<section>
<h1>Blogroll</h1>
<ul>
<li><a href="http://ozlabs.org" target="_blank">OzLabs</a></li>
</ul>
</section>
<section>
<h1>Disclaimer</h1>
<div>
This blog represents the views of the individual authors, and doesn't necessarily represent IBM's positions, strategies or opinions. </div>
</section>
</aside> </div>
</div>
<footer role="contentinfo"><p>
Copyright © 2015–2023 OzLabs —
<span class="credit">Powered by <a href="http://getpelican.com">Pelican</a></span>
</p></footer>
</body>
</html>