-
Notifications
You must be signed in to change notification settings - Fork 7
/
Copy pathindex5.html
858 lines (789 loc) · 75.7 KB
/
index5.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
<!DOCTYPE html>
<head>
<meta charset="utf-8">
<title>Store Halfword Byte-Reverse Indexed</title>
<meta name="author" content="OzLabs">
<link href="https://sthbrx.github.io/rss.xml" type="application/rss+xml" rel="alternate"
title="Store Halfword Byte-Reverse Indexed RSS Feed" />
<!-- http://t.co/dKP3o1e -->
<meta name="HandheldFriendly" content="True">
<meta name="MobileOptimized" content="320">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link href="https://sthbrx.github.io/favicon.png" rel="icon">
<link href="https://sthbrx.github.io/theme/css/main.css" media="screen, projection"
rel="stylesheet" type="text/css">
<link href="//fonts.googleapis.com/css?family=PT+Serif:regular,italic,bold,bolditalic"
rel="stylesheet" type="text/css">
<link href="//fonts.googleapis.com/css?family=PT+Sans:regular,italic,bold,bolditalic"
rel="stylesheet" type="text/css">
<script type="text/javascript">
document.addEventListener('DOMContentLoaded', function() {
var ts = document.createElement('span')
ts.className = 'toggle-sidebar'
ts = document.getElementById('content').appendChild(ts);
ts.addEventListener('click', function(e) {
e.preventDefault();
body = document.getElementsByTagName('body')[0];
bodyClasses = body.classList.toggle('collapse-sidebar');
});
var sections = document.querySelectorAll('aside.sidebar > section');
if (sections.length > 1) {
for (index = 0; index < sections.length; index++) {
section = sections[index];
if ((sections.length >= 3) && index % 3 === 0) {
section.classList.add("first");
}
var count = ((index +1) % 2) ? "odd" : "even";
section.classList.add(count);
}
}
if (sections.length >= 3) {
document.querySelector('aside.sidebar').classList.add('thirds');
}
});
</script>
<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-91189608-1', 'auto');
ga('send', 'pageview');
</script>
</head>
<body>
<header role="banner"><hgroup>
<h1><a href="https://sthbrx.github.io/">Store Halfword Byte-Reverse Indexed</a></h1>
<h2>A Power Technical Blog</h2>
</hgroup></header>
<nav role="navigation"><ul class="subscription" data-subscription="rss">
<li><a href="https://sthbrx.github.io/rss.xml" rel="subscribe-rss">RSS</a></li>
</ul>
<ul class="main-navigation">
<li >
<a href="https://sthbrx.github.io/category/cryptography.html">Cryptography</a>
</li>
<li >
<a href="https://sthbrx.github.io/category/development.html">Development</a>
</li>
<li >
<a href="https://sthbrx.github.io/category/education.html">Education</a>
</li>
<li >
<a href="https://sthbrx.github.io/category/openpower.html">OpenPOWER</a>
</li>
<li >
<a href="https://sthbrx.github.io/category/performance.html">Performance</a>
</li>
<li >
<a href="https://sthbrx.github.io/category/petitboot.html">Petitboot</a>
</li>
<li >
<a href="https://sthbrx.github.io/category/snowpatch.html">snowpatch</a>
</li>
<li >
<a href="https://sthbrx.github.io/category/virtualisation-and-emulation.html">Virtualisation and Emulation</a>
</li>
</ul></nav>
<div id="main">
<div id="content">
<div class="blog-index">
<article>
<header>
<h1 class="entry-title">
<a href="https://sthbrx.github.io/blog/2017/09/23/stupid-solutions-to-stupid-problems-hardcoding-your-ssh-key-in-the-kernel/">Stupid Solutions to Stupid Problems: Hardcoding Your SSH Key in the Kernel</a>
</h1>
<p class="meta">
<time datetime="2017-09-23T03:00:00+10:00" pubdate>Sat 23 September 2017</time> </p>
</header>
<div class="byline_index">
<span class="byline author vcard">
Posted by <span class="fn">
<a href="https://sthbrx.github.io/author/andrew-donnellan.html">Andrew Donnellan</a>
</span>
</span>
<time datetime="2017-09-23T03:00:00+10:00" pubdate>Sat 23 September 2017</time></div>
<div class="entry-content"><h2>The "problem"</h2>
<p>I'm currently working on firmware and kernel support for <a href="http://opencapi.org/">OpenCAPI</a> on POWER9.</p>
<p>I've recently been allocated a machine in the lab for development purposes. We use an internal IBM tool running on a secondary machine that triggers hardware initialisation procedures, then loads a specified <a href="https://github.com/open-power/skiboot">skiboot</a> firmware image, a kernel image, and a root file system directly into RAM. This allows us to get skiboot and Linux running without requiring the usual <a href="https://github.com/open-power/hostboot">hostboot</a> initialisation and gives us a lot of options for easier tinkering, so it's super-useful for our developers working on bringup.</p>
<p>When I got access to my machine, I figured out the necessary scripts, developed a workflow, and started fixing my code... so far, so good.</p>
<p>One day, I was trying to debug something and get logs off the machine using <code>ssh</code> and <code>scp</code>, when I got frustrated with having to repeatedly type in our ultra-secret, ultra-secure root password, <code>abc123</code>. So, I ran <code>ssh-copy-id</code> to copy over my public key, and all was good.</p>
<p>Until I rebooted the machine, when strangely, my key stopped working. It took me longer than it should have to realise that this is an obvious consequence of running entirely from an initrd that's reloaded every boot...</p>
<h2>The "solution"</h2>
<p>I mentioned something about this to Jono, my housemate/partner-in-stupid-ideas, one evening a few weeks ago. We decided that clearly, the best way to solve this problem was to hardcode my SSH public key in the kernel.</p>
<p>This would definitely be the easiest and most sensible way to solve the problem, as opposed to, say, just keeping my own copy of the root filesystem image. Or asking <a href="https://twitter.com/mikeyneuling">Mikey</a>, whose desk is three metres away from mine, whether he could use his write access to add my key to the image. Or just writing a wrapper around <a href="https://linux.die.net/man/1/sshpass">sshpass</a>...</p>
<p>One Tuesday afternoon, I was feeling bored...</p>
<h2>The approach</h2>
<p>The SSH daemon looks for authorised public keys in <code>~/.ssh/authorized_keys</code>, so we need to have a read of <code>/root/.ssh/authorized_keys</code> return a specified hard-coded string.</p>
<p>I did a bit of investigation. My first thought was to put some kind of hook inside whatever filesystem driver was being used for the root. After some digging, I found out that the filesystem type <code>rootfs</code>, as seen in <code>mount</code>, is actually backed by the <code>tmpfs</code> filesystem. I took a look around the <code>tmpfs</code> code for a while, but didn't see any way to hook in a fake file without a lot of effort - the <code>tmpfs</code> code wasn't exactly designed with this in mind.</p>
<p>I thought about it some more - what would be the easiest way to create a file such that it just returns a string?</p>
<p>Then I remembered sysfs, the filesystem normally mounted at <code>/sys</code>, which is used by various kernel subsystems to expose configuration and debugging information to userspace in the form of files. The sysfs API allows you to define a file and specify callbacks to handle reads and writes to the file.</p>
<p>That got me thinking - could I create a file in <code>/sys</code>, and then use a <a href="https://unix.stackexchange.com/questions/198590/what-is-a-bind-mount">bind mount</a> to have that file appear where I need it in <code>/root/.ssh/authorized_keys</code>? This approach seemed fairly straightforward, so I decided to give it a try.</p>
<p>First up, creating a pseudo-file. It had been a while since the last time I'd used the sysfs API...</p>
<h2>sysfs</h2>
<p>The sysfs pseudo file system was first introduced in Linux 2.6, and is generally used for exposing system and device information.</p>
<p>Per the <a href="https://www.kernel.org/doc/Documentation/filesystems/sysfs.txt">sysfs documentation</a>, sysfs is tied in very closely with the <a href="https://www.kernel.org/doc/Documentation/kobject.txt">kobject</a> infrastructure. sysfs exposes kobjects as directories, containing "attributes" represented as files. The kobject infrastructure provides a way to define kobjects representing entities (e.g. devices) and ksets which define collections of kobjects (e.g. devices of a particular type).</p>
<p>Using kobjects you can do lots of fancy things such as sending events to userspace when devices are hotplugged - but that's all out of the scope of this post. It turns out there's some fairly straightforward wrapper functions if all you want to do is create a kobject just to have a simple directory in sysfs.</p>
<div class="highlight"><pre><span></span><code><span class="cp">#include</span><span class="w"> </span><span class="cpf"><linux/kobject.h></span>
<span class="k">static</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">__init</span><span class="w"> </span><span class="nf">ssh_key_init</span><span class="p">(</span><span class="kt">void</span><span class="p">)</span>
<span class="p">{</span>
<span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">kobject</span><span class="w"> </span><span class="o">*</span><span class="n">ssh_kobj</span><span class="p">;</span>
<span class="w"> </span><span class="n">ssh_kobj</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">kobject_create_and_add</span><span class="p">(</span><span class="s">"ssh"</span><span class="p">,</span><span class="w"> </span><span class="nb">NULL</span><span class="p">);</span>
<span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="o">!</span><span class="n">ssh_kobj</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="n">pr_err</span><span class="p">(</span><span class="s">"SSH: kobject creation failed!</span><span class="se">\n</span><span class="s">"</span><span class="p">);</span>
<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="o">-</span><span class="n">ENOMEM</span><span class="p">;</span>
<span class="w"> </span><span class="p">}</span>
<span class="p">}</span>
<span class="n">late_initcall</span><span class="p">(</span><span class="n">ssh_key_init</span><span class="p">);</span>
</code></pre></div>
<p>This creates and adds a kobject called <code>ssh</code>. And just like that, we've got a directory in <code>/sys/ssh/</code>!</p>
<p>The next thing we have to do is define a sysfs attribute for our <code>authorized_keys</code> file. sysfs provides a framework for subsystems to define their own custom types of attributes with their own metadata - but for our purposes, we'll use the generic <code>bin_attribute</code> attribute type.</p>
<div class="highlight"><pre><span></span><code><span class="cp">#include</span><span class="w"> </span><span class="cpf"><linux/sysfs.h></span>
<span class="k">const</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="n">key</span><span class="p">[]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"PUBLIC KEY HERE..."</span><span class="p">;</span>
<span class="k">static</span><span class="w"> </span><span class="kt">ssize_t</span><span class="w"> </span><span class="nf">show_key</span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">file</span><span class="w"> </span><span class="o">*</span><span class="n">file</span><span class="p">,</span><span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">kobject</span><span class="w"> </span><span class="o">*</span><span class="n">kobj</span><span class="p">,</span>
<span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">bin_attribute</span><span class="w"> </span><span class="o">*</span><span class="n">bin_attr</span><span class="p">,</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="o">*</span><span class="n">to</span><span class="p">,</span>
<span class="w"> </span><span class="n">loff_t</span><span class="w"> </span><span class="n">pos</span><span class="p">,</span><span class="w"> </span><span class="kt">size_t</span><span class="w"> </span><span class="n">count</span><span class="p">)</span>
<span class="p">{</span>
<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="n">memory_read_from_buffer</span><span class="p">(</span><span class="n">to</span><span class="p">,</span><span class="w"> </span><span class="n">count</span><span class="p">,</span><span class="w"> </span><span class="o">&</span><span class="n">pos</span><span class="p">,</span><span class="w"> </span><span class="n">key</span><span class="p">,</span><span class="w"> </span><span class="n">bin_attr</span><span class="o">-></span><span class="n">size</span><span class="p">);</span>
<span class="p">}</span>
<span class="k">static</span><span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">bin_attribute</span><span class="w"> </span><span class="n">authorized_keys_attr</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="p">.</span><span class="n">attr</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="p">.</span><span class="n">name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"authorized_keys"</span><span class="p">,</span><span class="w"> </span><span class="p">.</span><span class="n">mode</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mo">0444</span><span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="p">.</span><span class="n">read</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">show_key</span><span class="p">,</span>
<span class="w"> </span><span class="p">.</span><span class="n">size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="k">sizeof</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
<span class="p">};</span>
</code></pre></div>
<p>We provide a simple callback, <code>show_key()</code>, that copies the key string into the file's buffer, and we put it in a <code>bin_attribute</code> with the appropriate name, size and permissions.</p>
<p>To actually add the attribute, we put the following in <code>ssh_key_init()</code>:</p>
<div class="highlight"><pre><span></span><code><span class="kt">int</span><span class="w"> </span><span class="n">rc</span><span class="p">;</span>
<span class="n">rc</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">sysfs_create_bin_file</span><span class="p">(</span><span class="n">ssh_kobj</span><span class="p">,</span><span class="w"> </span><span class="o">&</span><span class="n">authorized_keys_attr</span><span class="p">);</span>
<span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">rc</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="n">pr_err</span><span class="p">(</span><span class="s">"SSH: sysfs creation failed, rc %d</span><span class="se">\n</span><span class="s">"</span><span class="p">,</span><span class="w"> </span><span class="n">rc</span><span class="p">);</span>
<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="n">rc</span><span class="p">;</span>
<span class="p">}</span>
</code></pre></div>
<p>Woo, we've now got <code>/sys/ssh/authorized_keys</code>! Time to move on to the bind mount.</p>
<h2>Mounting</h2>
<p>Now that we've got a directory with the key file in it, it's time to figure out the bind mount.</p>
<p>Because I had no idea how any of the file system code works, I started off by running <code>strace</code> on <code>mount --bind ~/tmp1 ~/tmp2</code> just to see how the userspace <code>mount</code> tool uses the <code>mount</code> syscall to request the bind mount.</p>
<div class="highlight"><pre><span></span><code><span class="n">execve</span><span class="p">(</span><span class="s">"/bin/mount"</span><span class="p">,</span><span class="w"> </span><span class="p">[</span><span class="s">"mount"</span><span class="p">,</span><span class="w"> </span><span class="s">"--bind"</span><span class="p">,</span><span class="w"> </span><span class="s">"/home/ajd/tmp1"</span><span class="p">,</span><span class="w"> </span><span class="s">"/home/ajd/tmp2"</span><span class="p">],</span><span class="w"> </span><span class="p">[</span><span class="cm">/* 18 vars */</span><span class="p">])</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span>
<span class="p">...</span>
<span class="n">mount</span><span class="p">(</span><span class="s">"/home/ajd/tmp1"</span><span class="p">,</span><span class="w"> </span><span class="s">"/home/ajd/tmp2"</span><span class="p">,</span><span class="w"> </span><span class="mh">0x18b78bf00</span><span class="p">,</span><span class="w"> </span><span class="n">MS_MGC_VAL</span><span class="o">|</span><span class="n">MS_BIND</span><span class="p">,</span><span class="w"> </span><span class="nb">NULL</span><span class="p">)</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span>
</code></pre></div>
<p>The first and second arguments are the source and target paths respectively. The third argument, looking at the signature of the <code>mount</code> syscall, is a pointer to a string with the file system type. Because this is a bind mount, the type is irrelevant (upon further digging, it turns out that this particular pointer is to the string "none").</p>
<p>The fourth argument is where we specify the flags bitfield. <code>MS_MGC_VAL</code> is a magic value that was required before Linux 2.4 and can now be safely ignored. <code>MS_BIND</code>, as you can probably guess, signals that we want a bind mount.</p>
<p>(The final argument is used to pass file system specific data - as you can see it's ignored here.)</p>
<p>Now, how is the syscall actually handled on the kernel side? The answer is found in <a href="http://elixir.free-electrons.com/linux/latest/source/fs/namespace.c#L2969">fs/namespace.c</a>.</p>
<div class="highlight"><pre><span></span><code><span class="n">SYSCALL_DEFINE5</span><span class="p">(</span><span class="n">mount</span><span class="p">,</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="n">__user</span><span class="w"> </span><span class="o">*</span><span class="p">,</span><span class="w"> </span><span class="n">dev_name</span><span class="p">,</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="n">__user</span><span class="w"> </span><span class="o">*</span><span class="p">,</span><span class="w"> </span><span class="n">dir_name</span><span class="p">,</span>
<span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="n">__user</span><span class="w"> </span><span class="o">*</span><span class="p">,</span><span class="w"> </span><span class="n">type</span><span class="p">,</span><span class="w"> </span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">long</span><span class="p">,</span><span class="w"> </span><span class="n">flags</span><span class="p">,</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="n">__user</span><span class="w"> </span><span class="o">*</span><span class="p">,</span><span class="w"> </span><span class="n">data</span><span class="p">)</span>
<span class="p">{</span>
<span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">ret</span><span class="p">;</span>
<span class="w"> </span><span class="cm">/* ... copy parameters from userspace memory ... */</span>
<span class="w"> </span><span class="n">ret</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">do_mount</span><span class="p">(</span><span class="n">kernel_dev</span><span class="p">,</span><span class="w"> </span><span class="n">dir_name</span><span class="p">,</span><span class="w"> </span><span class="n">kernel_type</span><span class="p">,</span><span class="w"> </span><span class="n">flags</span><span class="p">,</span><span class="w"> </span><span class="n">options</span><span class="p">);</span>
<span class="w"> </span><span class="cm">/* ... cleanup ... */</span>
<span class="p">}</span>
</code></pre></div>
<p>So in order to achieve the same thing from within the kernel, we just call <code>do_mount()</code> with exactly the same parameters as the syscall uses:</p>
<div class="highlight"><pre><span></span><code><span class="n">rc</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">do_mount</span><span class="p">(</span><span class="s">"/sys/ssh"</span><span class="p">,</span><span class="w"> </span><span class="s">"/root/.ssh"</span><span class="p">,</span><span class="w"> </span><span class="s">"sysfs"</span><span class="p">,</span><span class="w"> </span><span class="n">MS_BIND</span><span class="p">,</span><span class="w"> </span><span class="nb">NULL</span><span class="p">);</span>
<span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">rc</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="n">pr_err</span><span class="p">(</span><span class="s">"SSH: bind mount failed, rc %d</span><span class="se">\n</span><span class="s">"</span><span class="p">,</span><span class="w"> </span><span class="n">rc</span><span class="p">);</span>
<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="n">rc</span><span class="p">;</span>
<span class="p">}</span>
</code></pre></div>
<p>...and we're done, right? Not so fast:</p>
<div class="highlight"><pre><span></span><code><span class="n">SSH</span><span class="o">:</span><span class="w"> </span><span class="n">bind</span><span class="w"> </span><span class="n">mount</span><span class="w"> </span><span class="n">failed</span><span class="o">,</span><span class="w"> </span><span class="n">rc</span><span class="w"> </span><span class="o">-</span><span class="mi">2</span>
</code></pre></div>
<p>-2 is <code>ENOENT</code> - no such file or directory. For some reason, we can't find <code>/sys/ssh</code>... of course, that would be because even though we've created the sysfs entry, we haven't actually mounted sysfs on <code>/sys</code>.</p>
<div class="highlight"><pre><span></span><code><span class="n">rc</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">do_mount</span><span class="p">(</span><span class="s">"sysfs"</span><span class="p">,</span><span class="w"> </span><span class="s">"/sys"</span><span class="p">,</span><span class="w"> </span><span class="s">"sysfs"</span><span class="p">,</span>
<span class="w"> </span><span class="n">MS_NOSUID</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="n">MS_NOEXEC</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="n">MS_NODEV</span><span class="p">,</span><span class="w"> </span><span class="nb">NULL</span><span class="p">);</span>
</code></pre></div>
<p>At this point, my key worked!</p>
<p>Note that this requires that your root file system has an empty directory created at <code>/sys</code> to be the mount point. Additionally, in a typical Linux distribution environment (as opposed to my hardware bringup environment), your initial root file system will contain an init script that mounts your real root file system somewhere and calls <code>pivot_root()</code> to switch to the new root file system. At that point, the bind mount won't be visible from children processes using the new root - I think this could be worked around but would require some effort.</p>
<h2>Kconfig</h2>
<p>The final piece of the puzzle is building our new code into the kernel image.</p>
<p>To allow us to switch this important functionality on and off, I added a config option to <code>fs/Kconfig</code>:</p>
<div class="highlight"><pre><span></span><code><span class="nv">config</span><span class="w"> </span><span class="nv">SSH_KEY</span>
<span class="w"> </span><span class="nv">bool</span><span class="w"> </span><span class="s2">"Andrew's dumb SSH key hack"</span>
<span class="w"> </span><span class="nv">default</span><span class="w"> </span><span class="nv">y</span>
<span class="w"> </span><span class="nv">help</span>
<span class="w"> </span><span class="nv">Hardcode</span><span class="w"> </span><span class="nv">an</span><span class="w"> </span><span class="nv">SSH</span><span class="w"> </span><span class="nv">key</span><span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="o">/</span><span class="nv">root</span><span class="o">/</span>.<span class="nv">ssh</span><span class="o">/</span><span class="nv">authorized_keys</span>.
<span class="w"> </span><span class="nv">This</span><span class="w"> </span><span class="nv">is</span><span class="w"> </span><span class="nv">a</span><span class="w"> </span><span class="nv">stupid</span><span class="w"> </span><span class="nv">idea</span>.<span class="w"> </span><span class="k">If</span><span class="w"> </span><span class="nv">unsure</span>,<span class="w"> </span><span class="nv">say</span><span class="w"> </span><span class="nv">N</span>.
</code></pre></div>
<p>This will show up in <code>make menuconfig</code> under the <code>File systems</code> menu.</p>
<p>And in <code>fs/Makefile</code>:</p>
<div class="highlight"><pre><span></span><code><span class="nv">obj-$(CONFIG_SSH_KEY)</span><span class="w"> </span><span class="o">+=</span><span class="w"> </span>ssh_key.o
</code></pre></div>
<p>If <code>CONFIG_SSH_KEY</code> is set to <code>y</code>, <code>obj-$(CONFIG_SSH_KEY)</code> evaluates to <code>obj-y</code> and thus <code>ssh-key.o</code> gets compiled. Conversely, <code>obj-n</code> is completely ignored by the build system.</p>
<p>I thought I was all done... then <a href="https://twitter.com/mramboar">Andrew</a> suggested I make the contents of the key configurable, and I had to oblige. Conveniently, Kconfig options can also be strings:</p>
<div class="highlight"><pre><span></span><code><span class="nv">config</span><span class="w"> </span><span class="nv">SSH_KEY_VALUE</span>
<span class="w"> </span><span class="nv">string</span><span class="w"> </span><span class="s2">"Value for SSH key"</span>
<span class="w"> </span><span class="nv">depends</span><span class="w"> </span><span class="nv">on</span><span class="w"> </span><span class="nv">SSH_KEY</span>
<span class="w"> </span><span class="nv">help</span>
<span class="w"> </span><span class="nv">Enter</span><span class="w"> </span><span class="nv">in</span><span class="w"> </span><span class="nv">the</span><span class="w"> </span><span class="nv">content</span><span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="o">/</span><span class="nv">root</span><span class="o">/</span>.<span class="nv">ssh</span><span class="o">/</span><span class="nv">authorized_keys</span>.
</code></pre></div>
<p>Including the string in the C file is as simple as:</p>
<div class="highlight"><pre><span></span><code><span class="k">const</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="n">key</span><span class="p">[]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">CONFIG_SSH_KEY_VALUE</span><span class="p">;</span>
</code></pre></div>
<p>And there we have it, a nicely configurable albeit highly limited kernel SSH backdoor!</p>
<h2>Conclusion</h2>
<p>I've put the <a href="https://github.com/ajdlinux/linux/commit/052c0cb7296f7510fd482fecbe572b641c29239f">full code</a> up on GitHub for perusal. Please don't use it, I will be extremely disappointed in you if you do.</p>
<p>Thanks to Jono for giving me stupid ideas, and the rest of OzLabs for being very angry when they saw the disgusting things I was doing.</p>
<p>Comments and further stupid suggestions welcome!</p></div>
</article>
<article>
<header>
<h1 class="entry-title">
<a href="https://sthbrx.github.io/blog/2017/09/22/ncsi-nice-network-youve-got-there/">NCSI - Nice Network You've Got There</a>
</h1>
<p class="meta">
<time datetime="2017-09-22T10:08:00+10:00" pubdate>Fri 22 September 2017</time> </p>
</header>
<div class="byline_index">
<span class="byline author vcard">
Posted by <span class="fn">
<a href="https://sthbrx.github.io/author/samuel-mendoza-jonas.html">Samuel Mendoza-Jonas</a>
</span>
</span>
<time datetime="2017-09-22T10:08:00+10:00" pubdate>Fri 22 September 2017</time></div>
<div class="entry-content"><p>A neat piece of kernel code dropped into my lap recently, and as a way of
processing having to inject an entire network stack into by brain in
less-than-ideal time I thought we'd have a look at it here: NCSI!</p>
<h2>NCSI - Not the TV Show</h2>
<p>NCSI stands for Network Controller Sideband Interface, and put most simply it
is a way for a management controller (eg. a BMC like those found on our OpenPOWER
machines) to share a single physical network interface with a host machine.
Instead of two distinct network interfaces you plug in a single cable and both
the host and the BMC have network connectivity.</p>
<p>NCSI-capable network controllers achieve this by filtering network traffic as
it arrives and determining if it is host- or BMC-bound. To know how to do this
the BMC needs to tell the network controller what to look out for, and from a
Linux driver perspective this the focus of the NCSI protocol.</p>
<p><img alt="NCSI Overview" src="/images/sammj/ncsi_overview.png"></p>
<h2>Hi My Name Is 70:e2:84:14:24:a1</h2>
<p>The major components of what NCSI helps facilitate are:</p>
<ul>
<li>Network Controllers, known as 'Packages' in this context. There may be multiple
separate packages which contain one or more Channels.</li>
<li>Channels, most easily thought of as the individual physical network interfaces.
If a package is the network card, channels are the individual network jacks. (Somewhere a pedant's head is spinning in circles).</li>
<li>Management Controllers, or our BMC, with their own network interfaces. Hypothetically there can be multiple
management controllers in a single NCSI system, but I've not come across such
a setup yet.</li>
</ul>
<p>NCSI is the medium and protocol via which these components communicate.</p>
<p><img alt="NCSI Packages" src="/images/sammj/ncsi_packages.png"></p>
<p>The interface between Management Controller and one or more
Packages carries both general network traffic to/from the Management
Controller as well as NCSI traffic between the Management Controller
and the Packages & Channels. Management traffic is differentiated from
regular traffic via the inclusion of a special NCSI tag inserted
in the Ethernet frame header.
These management commands are used to discover and configure the state of the
NCSI packages and channels.</p>
<p>If a BMC's network interface is configured to use NCSI, as soon as the interface
is brought up NCSI gets to work finding and configuring a usable channel.
The NCSI driver at first glance is an intimidating combination of state machines
and packet handlers, but with enough coffee it can be represented like this:</p>
<p><img alt="NCSI State Diagram" src="/images/sammj/ncsi_states.png"></p>
<p>Without getting into the nitty gritty details the overall process for configuring
a channel enough to get packets flowing is fairly straightforward:</p>
<ul>
<li>Find available packages.</li>
<li>Find each package's available channels.</li>
<li>(At least in the Linux driver) select a channel with link.</li>
<li>Put this channel into the Initial Config State.
The Initial Config State is where all the useful configuration occurs. Here we
find out what the selected channel is capable of and its current configuration,
and set it up to recognise the traffic we're interested in. The first and most
basic way of doing this is configuring the channel to filter traffic based on
our MAC address.</li>
<li>Enable the channel and let the packets flow.</li>
</ul>
<p>At this point NCSI takes a back seat to normal network traffic, transmitting
a "Get Link Status" packet at regular intervals to monitor the channel.</p>
<h2>AEN Packets</h2>
<p>Changes can occur from the package side too; the NCSI package communicates these
back to the BMC with Asynchronous Event Notification (AEN) packets. As the name
suggests these can occur at any time and the driver needs to catch and handle these.
There are different types but they essentially boil down to changes in link state,
telling the BMC the channel needs to be reconfigured, or to select a different
channel.
These are only transmitted once and no effort is made to recover lost AEN packets -
another good reason for the NCSI driver to periodically monitor the channel.</p>
<h2>Filtering</h2>
<p>Each channel can be configured to filter traffic based on MAC address,
broadcast traffic, multicast traffic, and VLAN tagging. Associated with each of
these filters is a <em>filter table</em> which can hold a finite number of entries.
In the case of the VLAN filter each channel could match against 15 different
VLAN IDs for example, but in practice the physical device will likely
support less. Indeed the popular BCM5718 controller supports only two!</p>
<p>This is where I dived into NCSI. The driver had a lot of the pieces for
configuring VLAN filters but none of it was actually hooked up in the configure
state, and didn't have a way of actually knowing which VLAN IDs were meant to be
configured on the interface. The bulk of that work appears in <a href="https://github.com/torvalds/linux/commit/21acf63013ed3d6fce3176cc34b74064052a31b4#diff-f391518f4e552724349be3589e00dfa7">this commit</a> where we take advantage of some useful network stack callbacks to get the VLAN configuration and set them during the configuration state. Getting <em>to</em> the configuration state at some arbitrary time and then managing to assign multiple IDs was the trickiest bit, and is something I'll be looking at simplifying in the future.</p>
<hr>
<p>NCSI! A neat way to give physically separate users access to a single network controller, and if it works right you won't notice it at all. I'll surely be spending more time here (fleshing out the driver's features, better error handling, and making the state machine a touch more readable to start, and I haven't even <em>mentioned</em> HWA), so watch this space!</p></div>
</article>
<article>
<header>
<h1 class="entry-title">
<a href="https://sthbrx.github.io/blog/2017/09/01/memcmp-for-power8-part-ii/">memcmp() for POWER8 - part II</a>
</h1>
<p class="meta">
<time datetime="2017-09-01T12:00:00+10:00" pubdate>Fri 01 September 2017</time> </p>
</header>
<div class="byline_index">
<span class="byline author vcard">
Posted by <span class="fn">
<a href="https://sthbrx.github.io/author/cyril-bur.html">Cyril Bur</a>
</span>
</span>
<time datetime="2017-09-01T12:00:00+10:00" pubdate>Fri 01 September 2017</time></div>
<div class="entry-content"><p>This entry is a followup to part I which you should absolutely read
<a href="https://sthbrx.github.io/blog/2017/08/07/memcmp-for-power8/">here</a> before continuing
on.</p>
<h2>Where we left off</h2>
<p>We concluded that while a vectorised <code>memcmp()</code> is a win, there are
some cases where it won't quite perform.</p>
<h2>The overhead of enabling ALTIVEC</h2>
<p>In the kernel we explicitly don't touch ALTIVEC unless we need to,
this means that in the general case we can leave the userspace
registers in place and not have do anything to service a syscall for a
process.</p>
<p>This means that if we do want to use ALTIVEC in the kernel, there is
some setup that must be done. Notably, we must enable the facility (a
potentially time consuming move to MSR), save off the registers (if
userspace we using them) and an inevitable restore later on.</p>
<p>If all this needs to be done for a <code>memcmp()</code> in the order of tens of
bytes then it really wasn't worth it.</p>
<p>There are two reasons that <code>memcmp()</code> might go for a small number of
bytes, firstly and trivially detectable is simply that parameter n is
small. The other is harder to detect, if the memcmp() is going to fail
(return non zero) early then it also wasn't worth enabling ALTIVEC.</p>
<h2>Detecting early failures</h2>
<p>Right at the start of <code>memcmp()</code>, before enabling ALTIVEC, the first
64 bytes are checked using general purpose registers. Why the first 64
bytes, well why not? In a strange twist of fate 64 bytes happens to be
the amount of bytes in four ALTIVEC registers (128 bits per register,
so 16 bytes multiplied by 4) and by utter coincidence that happens to
be the stride of the ALTIVEC compare loop.</p>
<h2>What does this all look like</h2>
<p>Well unlike part I the results appear slightly less consistent across
three runs of measurement but there are some very key differences with
part I. The trends do appear to be the same across all three runs,
just less pronounced - why this is is unclear.</p>
<p>The difference between run two and run three clipped at deltas of
1000ns is interesting:
<img alt="Sample 2: Deltas below 1000ns" src="/images/power8_memcmp/v2deltas2-1000.png" title="Sample 2: Deltas below 1000ns"></p>
<p>vs</p>
<p><img alt="Sample 3: Deltas below 1000ns" src="/images/power8_memcmp/v2deltas3-1000.png" title="Sample 3: Deltas below 1000ns"></p>
<p>The results are similar except for a spike in the amount of deltas in
the unpatched kernel at around 600ns. This is not present in the first
sample (deltas1) of data. There are a number of reasons why this spike
could have appeared here, it is possible that the kernel or hardware
did something under the hood, prefetch could have brought deltas for a
<code>memcmp()</code> that would otherwise have yielded a greater delta into the
600ns range.</p>
<p>What these two graphs do both demonstrate quite clearly is that
optimisations down at the sub 100ns end have resulted in more sub
100ns deltas for the patched kernel, a significant win over the
original data. Zooming out and looking at a graph which includes
deltas up to 5000ns shows that the sub 100ns delta optimisations
haven't noticeably slowed the performance of long duration <code>memcmp()</code>,
<img alt="Samply 2: Deltas below 5000ns" src="/images/power8_memcmp/v2deltas2-5000.png" title="Sample 2: Deltas below 5000ns">.</p>
<h2>Conclusion</h2>
<p>The small amount of extra development effort has yielded tangible
results in reducing the low end <code>memcmp()</code> times. This second round of
data collection and performance analysis only confirms the that for
any significant amount of comparison, a vectorised loop is
significantly quicker.</p>
<p>The results obtained here show no downside to adopting this approach
for all power8 and onwards chips as this new version of the patch
solves the performance regression for small compares.</p></div>
</article>
<article>
<header>
<h1 class="entry-title">
<a href="https://sthbrx.github.io/blog/2017/08/07/memcmp-for-power8/">memcmp() for POWER8</a>
</h1>
<p class="meta">
<time datetime="2017-08-07T12:00:00+10:00" pubdate>Mon 07 August 2017</time> </p>
</header>
<div class="byline_index">
<span class="byline author vcard">
Posted by <span class="fn">
<a href="https://sthbrx.github.io/author/cyril-bur.html">Cyril Bur</a>
</span>
</span>
<time datetime="2017-08-07T12:00:00+10:00" pubdate>Mon 07 August 2017</time></div>
<div class="entry-content"><h2>Userspace</h2>
<p>When writing C programs in userspace there is libc which does so much
of the heavy lifting. One important thing libc provides is portability
in performing syscalls, that is, you don't need to know the
architectural details of performing a syscall on each architecture
your program might be compiled for. Another important feature that
libc provides for the average userspace programmer is highly optimised
routines to do things that are usually performance critical. It would
be extremely inefficient for each userspace programmer if they had to
implement even the naive version of these functions let alone
optimised versions. Let us take <code>memcmp()</code> for example, I could
trivially implement this in C like:</p>
<div class="highlight"><pre><span></span><code><span class="kt">int</span><span class="w"> </span><span class="nf">memcmp</span><span class="p">(</span><span class="kt">uint8_t</span><span class="w"> </span><span class="o">*</span><span class="n">p1</span><span class="p">,</span><span class="w"> </span><span class="kt">uint8_t</span><span class="w"> </span><span class="o">*</span><span class="n">p2</span><span class="p">,</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">n</span><span class="p">)</span>
<span class="p">{</span>
<span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">i</span><span class="p">;</span>
<span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="n">i</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="n">n</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">++</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">p1</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="n">p2</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>
<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="mi">-1</span><span class="p">;</span>
<span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">p1</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="w"> </span><span class="o">></span><span class="w"> </span><span class="n">p2</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>
<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="mi">1</span><span class="p">;</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="mi">0</span><span class="p">;</span>
<span class="p">}</span>
</code></pre></div>
<p>However, while it is incredibly portable it is simply not going to
perform, which is why the nice people who write libc have highly
optimised ones in assembly for each architecture.</p>
<h2>Kernel</h2>
<p>When writing code for the Linux kernel, there isn't the luxury of a
fully featured libc since it expects (and needs) to be in userspace,
therefore we need to implement the features we need ourselves. Linux
doesn't need all the features but something like <code>memcmp()</code> is
definitely a requirement.</p>
<p>There have been some recent optimisations in <a href="https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/powerpc/powerpc64/power8/memcmp.S;h=46b9c0067ad7cd74a36c4800ebfe03eb1be0311e;hb=dec4a7105edcdbabdcac5f358f5bc5dca4f4ed1b" title="power8 optimised memcmp">glibc</a> from which the
kernel could benefit too! The question to be asked is, does the glibc
optimised <code>power8_memcmp()</code> actually go faster or is it all smoke and
mirrors?</p>
<h2>Benchmarking <code>memcmp()</code></h2>
<p>With things like <code>memcmp()</code> it is actually quite easy to choose
datasets which can make any implementation look good. For example; the
new <code>power8_memcmp()</code> makes use of the vector unit of the power8
processor, in order to do so in the kernel there must be a small
amount of setup code so that the rest of the kernel knows that the
vector unit has been used and it correctly saves and restores the
userspace vector registers. This means that <code>power8_memcmp()</code> has a
slightly larger overhead than the current one, so for small compares
or compares which are different early on then the newer 'faster'
<code>power8_memcmp()</code> might actually not perform as well. For any kind of
large compare however, using the vector unit should outperform a CPU
register load and compare loop. It is for this reason that I wanted to
avoid using micro benchmarks and use a 'real world' test as much as
possible.</p>
<p>The biggest user of <code>memcmp()</code> in the kernel, at least on POWER is Kernel
Samepage Merging (KSM). KSM provides code to inspect all the pages of
a running system to determine if they're identical and deduplicate
them if possible. This kind of feature allows for memory overcommit
when used in a KVM host environment as guest kernels are likely to
have a lot of similar, readonly pages which can be merged with no
overhead afterwards. In order to determine if the pages are the same
KSM must do a lot of page sized <code>memcmp()</code>.</p>
<h2>Performance</h2>
<p>Performing a lot of page sized <code>memcmp()</code> is the one flaw with this
test, the sizes of the <code>memcmp()</code> don't vary, hopefully the data will be
'random' enough that we can still observe differences in the two
approaches.</p>
<p>My approach for testing involved getting the delta of <code>ktime_get()</code>
across calls to <code>memcmp()</code> in <code>memcmp_pages()</code> (mm/ksm.c). This actually
generated massive amounts of data, so, for consistency the following
analysis is performed on the first 400MB of deltas collected.</p>
<p>The host was compiled with <code>powernv_defconfig</code> and run out of a
ramdisk. For consistency the host was rebooted between each run so as
to not have any previous tests affect the next. The host was rebooted
a total of six times, the first three with my 'patched'
<code>power8_memcmp()</code> kernel was booted the second three times with just
my data collection patch applied, the 'vanilla' kernel. Both
kernels are based off <code>4.13-rc3</code>.</p>
<p>Each boot the following script was run and the resulting deltas file
saved somewhere before reboot. The command line argument was always
15.</p>
<div class="highlight"><pre><span></span><code><span class="ch">#!/bin/sh</span>
ppc64_cpu<span class="w"> </span>--smt<span class="o">=</span>off
<span class="c1">#Host actually boots with ksm off but be sure</span>
<span class="nb">echo</span><span class="w"> </span><span class="m">0</span><span class="w"> </span>><span class="w"> </span>/sys/kernel/mm/ksm/run
<span class="c1">#Scan a lot of pages</span>
<span class="nb">echo</span><span class="w"> </span><span class="m">999999</span><span class="w"> </span>><span class="w"> </span>/sys/kernel/mm/ksm/pages_to_scan
<span class="nb">echo</span><span class="w"> </span><span class="s2">"Starting QEMUs"</span>
<span class="nv">i</span><span class="o">=</span><span class="m">0</span>
<span class="k">while</span><span class="w"> </span><span class="o">[</span><span class="w"> </span><span class="s2">"</span><span class="nv">$i</span><span class="s2">"</span><span class="w"> </span>-lt<span class="w"> </span><span class="s2">"</span><span class="nv">$1</span><span class="s2">"</span><span class="w"> </span><span class="o">]</span><span class="w"> </span><span class="p">;</span><span class="w"> </span><span class="k">do</span>
<span class="w"> </span>qemu-system-ppc64<span class="w"> </span>-smp<span class="w"> </span><span class="m">1</span><span class="w"> </span>-m<span class="w"> </span>1G<span class="w"> </span>-nographic<span class="w"> </span>-vga<span class="w"> </span>none<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>-machine<span class="w"> </span>pseries,accel<span class="o">=</span>kvm,kvm-type<span class="o">=</span>HV<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>-kernel<span class="w"> </span>guest.kernel<span class="w"> </span>-initrd<span class="w"> </span>guest.initrd<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>-monitor<span class="w"> </span>pty<span class="w"> </span>-serial<span class="w"> </span>pty<span class="w"> </span><span class="p">&</span>
<span class="w"> </span><span class="nv">i</span><span class="o">=</span><span class="k">$(</span>expr<span class="w"> </span><span class="nv">$i</span><span class="w"> </span>+<span class="w"> </span><span class="m">1</span><span class="k">)</span><span class="p">;</span>
<span class="k">done</span>
<span class="nb">echo</span><span class="w"> </span><span class="s2">"Letting all the VMs boot"</span>
sleep<span class="w"> </span><span class="m">30</span>
<span class="nb">echo</span><span class="w"> </span><span class="s2">"Turning KSM om"</span>
<span class="nb">echo</span><span class="w"> </span><span class="m">1</span><span class="w"> </span>><span class="w"> </span>/sys/kernel/mm/ksm/run
<span class="nb">echo</span><span class="w"> </span><span class="s2">"Letting KSM do its thing"</span>
sleep<span class="w"> </span>2m
<span class="nb">echo</span><span class="w"> </span><span class="m">0</span><span class="w"> </span>><span class="w"> </span>/sys/kernel/mm/ksm/run
dd<span class="w"> </span><span class="k">if</span><span class="o">=</span>/sys/kernel/debug/ksm/memcmp_deltas<span class="w"> </span><span class="nv">of</span><span class="o">=</span>deltas<span class="w"> </span><span class="nv">bs</span><span class="o">=</span><span class="m">4096</span><span class="w"> </span><span class="nv">count</span><span class="o">=</span><span class="m">100</span>
</code></pre></div>
<p>The guest kernel was a <code>pseries_le_defconfig</code> <code>4.13-rc3</code> with the same
ramdisk the host used. It booted to the login prompt and was left to
idle.</p>
<h2>Analysis</h2>
<p>A variety of histograms were then generated in an attempt to see how
the behaviour of <code>memcmp()</code> changed between the two implementations.
It should be noted here that the y axis in the following graphs is a
log scale as there were a lot of small deltas. The first observation
is that the vanilla kernel had more smaller deltas, this is made
particularly evident by the 'tally' points which are a running total
of all deltas with less than the tally value.</p>
<p><img alt="Sample 1 - Deltas below 200ns" src="/images/power8_memcmp/deltas1-200.png" title="Sample 1: Deltas below 200ns">
Graph 1 depicting the vanilla kernel having a greater amount of small
(sub 20ns) deltas than the patched kernel. The green points rise
faster (left to right) and higher than the yellow points.</p>
<p>Still looking at the tallies, <a href="/images/power8_memcmp/deltas1-200.png" title="Sample 1: Deltas below 200ns">graph 1</a> also shows that the tally
of deltas is very close by the 100ns mark, which means that the
overhead of <code>power8_memcmp()</code> is not too great.</p>
<p>The problem with looking at only deltas under 200ns is that the
performance results we want, that is, the difference between the
algorithms is being masked by things like cache effects. To avoid this
problem is may be wise to look at longer running (larger delta)
<code>memcmp()</code> calls.</p>
<p>The following graph plots all deltas below 5000ns - still relatively
short calls to <code>memcmp()</code> but an interesting trend emerges:
<img alt="Sample 1 - Deltas below 5000ns" src="/images/power8_memcmp/deltas1-5000.png" title="Sample 1: Deltas below 5000ns">
Graph 2 shows that above 500ns the blue (patched kernel) points appear
to have all shifted left with respect to the purple (vanilla kernel)
points. This shows that for any <code>memcmp()</code> which will take more than
500ns to get a result it is favourable to use <code>power8_memcmp()</code> and it
is only detrimental to use <code>power8_memcmp()</code> if the time will be
under 50ns (a conservative estimate).</p>
<p>It is worth noting that <a href="/images/power8_memcmp/deltas1-200.png" title="Sample 1: Deltas below 200ns">graph 1</a> and <a href="/images/power8_memcmp/deltas1-5000.png" title="Sample 1: Deltas below 5000ns">graph 2</a> are generated by
combining the first run of data collected from the vanilla and patched
kernels. All the deltas for both runs are can be viewed separately
<a href="/images/power8_memcmp/vanilla_deltas1.png" title="All vanilla deltas">here for vanilla</a> and <a href="/images/power8_memcmp/patched_deltas1.png" title="All patched deltas">here for patched</a>. Finally, the results
from the other four runs look very much identical and provide me with
a fair amount of confidence that these results make sense.</p>
<h2>Conclusions</h2>
<p>It is important to separate possible KSM optimisations with generic
<code>memcmp()</code> optimisations, for example, perhaps KSM shouldn't be
calling <code>memcmp()</code> if it suspects the first byte will differ. On the
other hand, things that <code>power8_memcmp()</code> could do (which it currently
doesn't) is check the length parameter and perhaps avoid the overhead
of enabling kernel vector if the compare is less than some small
amount of bytes.</p>
<p>It does seem like at least for the 'average case' glibcs
<code>power8_memcmp()</code> is an improvement over what we have now.</p>
<h2>Future work</h2>
<p>A second round of data collection and plotting of delta vs position of
first byte to differ should confirm these results, this would mean a
more invasive patch to KSM.</p></div>
</article>
<article>
<header>
<h1 class="entry-title">
<a href="https://sthbrx.github.io/blog/2017/07/17/xdp-on-power/">XDP on Power</a>
</h1>
<p class="meta">
<time datetime="2017-07-17T10:08:00+10:00" pubdate>Mon 17 July 2017</time> </p>
</header>
<div class="byline_index">
<span class="byline author vcard">
Posted by <span class="fn">
<a href="https://sthbrx.github.io/author/daniel-axtens.html">Daniel Axtens</a>
</span>
</span>
<time datetime="2017-07-17T10:08:00+10:00" pubdate>Mon 17 July 2017</time></div>
<div class="entry-content"><p>This post is a bit of a break from the standard IBM fare of this blog,
as I now work for Canonical. But I have a soft spot for Power from my
time at IBM - and Canonical officially supports 64-bit, little-endian
Power - so when I get a spare moment I try to make sure that cool,
officially-supported technologies work on Power <em>before</em> we end up
with a customer emergency! So, without further ado, this is the story
of XDP on Power.</p>
<h2>XDP</h2>
<p>eXpress Data Path (XDP) is a cool Linux technology to allow really
fast processing of network packets.</p>
<p>Normally in Linux, a packet is received by the network card, an SKB
(<a href="http://vger.kernel.org/~davem/skb.html">socket buffer</a>) is
allocated, and the packet is passed up through the networking stack.</p>
<p>This introduces an inescapable latency penalty: we have to allocate
some memory and copy stuff around. XDP allows some network cards and
drivers to process packets early - even before the allocation of the
SKB. This is much faster, and so has applications in DDOS mitigation
and other high-speed networking use-cases. The IOVisor project has
<a href="https://www.iovisor.org/technology/xdp">much more information</a> if you
want to learn more.</p>
<h2>eBPF</h2>
<p>XDP processing is done by an eBPF program. eBPF - the extended
Berkeley Packet Filter - is an in-kernel virtual machine with a
limited set of instructions. The kernel can statically validate eBPF
programs to ensure that they terminate and are memory safe. From this
it follows that the programs cannot be Turing-complete: they do not
have backward branches, so they cannot do fancy things like
loops. Nonetheless, they're surprisingly powerful for packet
processing and tracing. eBPF programs are translated into efficient
machine code using in-kernel JIT compilers on many platforms, and
interpreted on platforms that do not have a JIT. (Yes, there are
multiple JIT implementations in the kernel. I find this a terrifying
thought.)</p>
<p>Rather than requiring people to write raw eBPF programs, you can write
them in a somewhat-restricted subset of C, and use Clang's eBPF target
to translate them. This is super handy, as it gives you access to the
kernel headers - which define a number of useful data structures like
headers for various network protocols.</p>
<h2>Trying it</h2>
<p>There are a few really interesting project that are already up and
running that allow you to explore XDP without learning the innards of
both eBPF and the kernel networking stack. I explored the samples in
the <a href="https://github.com/iovisor/bcc">bcc compiler collection</a> and also
the samples from the <a href="https://github.com/netoptimizer/prototype-kernel/">netoptimizer/prototype-kernel repository</a>.</p>
<p>The easiest way to get started with these is with a virtual machine,
as recent virtio network drivers support XDP. If you are using Ubuntu,
you can use the <a href="https://help.ubuntu.com/lts/serverguide/cloud-images-and-uvtool.html">uvt-kvm
tooling</a>
to trivially set up a VM running Ubuntu Zesty on your local machine.</p>
<p>Once your VM is installed, you need to shut it down and edit the virsh XML. </p>
<p>You need 2 vCPUs (or more) and a virtio+vhost network card. You also
need to edit the 'interface' section and add the following snippet
(with thanks to the <a href="https://www.spinics.net/lists/xdp-newbies/msg00029.html">xdp-newbies
list</a>):</p>
<div class="highlight"><pre><span></span><code><span class="nt"><driver</span><span class="w"> </span><span class="na">name=</span><span class="s">'vhost'</span><span class="w"> </span><span class="na">queues=</span><span class="s">'4'</span><span class="nt">></span>
<span class="w"> </span><span class="nt"><host</span><span class="w"> </span><span class="na">tso4=</span><span class="s">'off'</span><span class="w"> </span><span class="na">tso6=</span><span class="s">'off'</span><span class="w"> </span><span class="na">ecn=</span><span class="s">'off'</span><span class="w"> </span><span class="na">ufo=</span><span class="s">'off'</span><span class="nt">/></span>
<span class="w"> </span><span class="nt"><guest</span><span class="w"> </span><span class="na">tso4=</span><span class="s">'off'</span><span class="w"> </span><span class="na">tso6=</span><span class="s">'off'</span><span class="w"> </span><span class="na">ecn=</span><span class="s">'off'</span><span class="w"> </span><span class="na">ufo=</span><span class="s">'off'</span><span class="nt">/></span>
<span class="nt"></driver></span>
</code></pre></div>
<p>(If you have more than 2 vCPUs, set the queues parameter to 2x the
number of vCPUs.)</p>
<p>Then, install a modern clang (we've had issues with 3.8 - I recommend
v4+), and the usual build tools.</p>
<p>I recommend testing with the prototype-kernel tools - the DDOS
prevention tool is a good demo. Then - on x86 - you just follow their
instructions. I'm not going to repeat that here.</p>
<h2>POWERful XDP</h2>
<p>What happens when you try this on Power? Regular readers of my posts
will know to expect some
<a href="https://sthbrx.github.io/blog/2017/02/13/high-power-lustre/">minor</a>
<a href="https://sthbrx.github.io/blog/2017/02/01/namd-on-nvlink/">hitches</a>.</p>
<p>XDP does not disappoint.</p>
<p>Firstly, the prototype-kernel repository <a href="https://github.com/netoptimizer/prototype-kernel/blob/master/kernel/samples/bpf/Makefile#L92">hard codes x86</a>
as the architecture for kernel headers. You need to change it for
powerpc.</p>
<p>Then, once you get the stuff compiled, and try to run it on a
current-at-time-of-writing Zesty kernel, you'll hit a massive debug
splat ending in:</p>
<div class="highlight"><pre><span></span><code><span class="mi">32</span><span class="p">:</span><span class="w"> </span><span class="p">(</span><span class="mi">61</span><span class="p">)</span><span class="w"> </span><span class="n">r1</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="o">*</span><span class="p">(</span><span class="n">u32</span><span class="w"> </span><span class="o">*</span><span class="p">)(</span><span class="n">r8</span><span class="w"> </span><span class="o">+</span><span class="mi">12</span><span class="p">)</span>
<span class="n">misaligned</span><span class="w"> </span><span class="n">packet</span><span class="w"> </span><span class="n">access</span><span class="w"> </span><span class="n">off</span><span class="w"> </span><span class="mi">0</span><span class="o">+</span><span class="mi">18</span><span class="o">+</span><span class="mi">12</span><span class="w"> </span><span class="n">size</span><span class="w"> </span><span class="mi">4</span>
<span class="n">load_bpf_file</span><span class="p">:</span><span class="w"> </span><span class="n">Permission</span><span class="w"> </span><span class="n">denied</span>
</code></pre></div>
<p>It turns out this is because in Ubuntu's Zesty kernel,
CONFIG_HAS_EFFICIENT_UNALIGNED_ACCESS is not set on ppc64el. Because
of that, the eBPF verifier will check that all loads are aligned - and
this load (part of checking some packet header) is not, and so the
verifier rejects the program. Unaligned access is not enabled because
the Zesty kernel is being compiled for CPU_POWER7 instead of
CPU_POWER8, and we don't have efficient unaligned access on POWER7.</p>
<p>As it turns out, IBM never released any officially supported Power7 LE
systems - LE was only ever supported on Power8. So, I <a href="https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1699627">filed a bug</a> and
<a href="https://lists.ubuntu.com/archives/kernel-team/2017-June/085074.html">sent a patch</a>
to build Zesty kernels for POWER8 instead, and that has been accepted
and will be part of the next stable update due real soon now.</p>
<p>Sure enough, if you install a kernel with that config change, you can
verify the XDP program and load it into the kernel!</p>
<p>If you have real powerpc hardware, that's enough to use XDP on Power!
Thanks to <a href="http://michael.ellerman.id.au/">Michael Ellerman</a>,
maintainer extraordinaire, for verifying this for me.</p>
<p>If - like me - you don't have ready access to Power hardware, you're
stuffed. You can't use qemu in TCG mode: to use XDP with a VM, you
need multi-queue support, which only exists in the vhost driver, which
is only available for KVM guests. Maybe IBM should release a developer
workstation. (Hint, hint!)</p>
<p>Overall, I was pleasantly surprised by how easy things were for people
with real ppc hardware - it's encouraging to see something not require
kernel changes!</p>
<p>eBPF and XDP are definitely growing technologies - as <a href="https://twitter.com/brendangregg/status/866078955530444800">Brendan Gregg notes</a>,
now is a good time to learn them! (And those on Power have no excuse
either!)</p></div>
</article>
<div class="pagination">
<a class="prev" href="https://sthbrx.github.io/index6.html">← Older</a>
<a class="next" href="https://sthbrx.github.io/index4.html">Newer →</a>
<br />
</div></div>
<aside class="sidebar">
<section>
<h1>Recent Posts</h1>
<ul id="recent_posts">
<li class="post">
<a href="https://sthbrx.github.io/blog/2023/08/07/going-out-on-a-limb-efficient-elliptic-curve-arithmetic-in-openssl/">Going out on a Limb: Efficient Elliptic Curve Arithmetic in OpenSSL</a>
</li>
<li class="post">
<a href="https://sthbrx.github.io/blog/2023/08/04/quirks-of-parsing-ssh-configs/">Quirks of parsing SSH configs</a>
</li>
<li class="post">
<a href="https://sthbrx.github.io/blog/2023/04/05/detecting-rootless-docker/">Detecting rootless Docker</a>
</li>
<li class="post">
<a href="https://sthbrx.github.io/blog/2023/04/04/dumb-bugs-the-pci-device-that-wasnt/">Dumb bugs: the PCI device that wasn't</a>
</li>
<li class="post">
<a href="https://sthbrx.github.io/blog/2023/03/24/dumb-bugs-when-a-date-breaks-booting-the-kernel/">Dumb bugs: When a date breaks booting the kernel</a>
</li>
</ul>
</section>
<section>
<h1>Categories</h1>
<ul id="recent_posts">
<li><a href="https://sthbrx.github.io/category/cryptography.html">Cryptography</a></li>
<li><a href="https://sthbrx.github.io/category/development.html">Development</a></li>
<li><a href="https://sthbrx.github.io/category/education.html">Education</a></li>
<li><a href="https://sthbrx.github.io/category/openpower.html">OpenPOWER</a></li>
<li><a href="https://sthbrx.github.io/category/performance.html">Performance</a></li>
<li><a href="https://sthbrx.github.io/category/petitboot.html">Petitboot</a></li>
<li><a href="https://sthbrx.github.io/category/snowpatch.html">snowpatch</a></li>
<li><a href="https://sthbrx.github.io/category/virtualisation-and-emulation.html">Virtualisation and Emulation</a></li>
</ul>
</section>
<section>
<h1>Tags</h1>
<a href="https://sthbrx.github.io/tag/ssh.html">ssh</a>, <a href="https://sthbrx.github.io/tag/docker.html">Docker</a>, <a href="https://sthbrx.github.io/tag/syzkaller.html">syzkaller</a>, <a href="https://sthbrx.github.io/tag/linux.html">linux</a>, <a href="https://sthbrx.github.io/tag/power8.html">power8</a>, <a href="https://sthbrx.github.io/tag/distro.html">distro</a>, <a href="https://sthbrx.github.io/tag/kernel.html">kernel</a>, <a href="https://sthbrx.github.io/tag/hardening.html">hardening</a>, <a href="https://sthbrx.github.io/tag/testing.html">testing</a>, <a href="https://sthbrx.github.io/tag/conferences.html">conferences</a>, <a href="https://sthbrx.github.io/tag/instruction-set-architecture.html">Instruction Set Architecture</a>, <a href="https://sthbrx.github.io/tag/openpower.html">openpower</a>, <a href="https://sthbrx.github.io/tag/firmware.html">firmware</a>, <a href="https://sthbrx.github.io/tag/goodposts.html">goodposts</a>, <a href="https://sthbrx.github.io/tag/realcontent.html">realcontent</a>, <a href="https://sthbrx.github.io/tag/madposting.html">madposting</a>, <a href="https://sthbrx.github.io/tag/op-test.html">op-test</a>, <a href="https://sthbrx.github.io/tag/qemu.html">qemu</a>, <a href="https://sthbrx.github.io/tag/pci.html">pci</a>, <a href="https://sthbrx.github.io/tag/sparseposting.html">sparseposting</a>, <a href="https://sthbrx.github.io/tag/petitboot.html">petitboot</a>, <a href="https://sthbrx.github.io/tag/security.html">security</a>, <a href="https://sthbrx.github.io/tag/vscode.html">vscode</a>, <a href="https://sthbrx.github.io/tag/code.html">code</a>, <a href="https://sthbrx.github.io/tag/openbmc.html">openbmc</a>, <a href="https://sthbrx.github.io/tag/ipmi.html">ipmi</a>, <a href="https://sthbrx.github.io/tag/opencapi.html">opencapi</a>, <a href="https://sthbrx.github.io/tag/openpower-summit.html">openpower summit</a>, <a href="https://sthbrx.github.io/tag/easyposts.html">easyposts</a>, <a href="https://sthbrx.github.io/tag/linuxboot.html">linuxboot</a>, <a href="https://sthbrx.github.io/tag/google.html">google</a>, <a href="https://sthbrx.github.io/tag/intel.html">intel</a>, <a href="https://sthbrx.github.io/tag/osfc.html">osfc</a>, <a href="https://sthbrx.github.io/tag/shortposts.html">shortposts</a>, <a href="https://sthbrx.github.io/tag/facebook.html">facebook</a>, <a href="https://sthbrx.github.io/tag/performance.html">performance</a>, <a href="https://sthbrx.github.io/tag/phoronix.html">phoronix</a>, <a href="https://sthbrx.github.io/tag/benchmarks.html">benchmarks</a>, <a href="https://sthbrx.github.io/tag/stupid-ideas.html">stupid ideas</a>, <a href="https://sthbrx.github.io/tag/network.html">network</a>, <a href="https://sthbrx.github.io/tag/power.html">power</a>, <a href="https://sthbrx.github.io/tag/xdp.html">xdp</a>, <a href="https://sthbrx.github.io/tag/networking.html">networking</a>, <a href="https://sthbrx.github.io/tag/remoteposts.html">remoteposts</a>, <a href="https://sthbrx.github.io/tag/ceph.html">ceph</a>, <a href="https://sthbrx.github.io/tag/raid.html">raid</a>, <a href="https://sthbrx.github.io/tag/storage.html">storage</a>, <a href="https://sthbrx.github.io/tag/erasure.html">erasure</a>, <a href="https://sthbrx.github.io/tag/lustre.html">lustre</a>, <a href="https://sthbrx.github.io/tag/hpc.html">hpc</a>, <a href="https://sthbrx.github.io/tag/nvlink.html">nvlink</a>, <a href="https://sthbrx.github.io/tag/namd.html">namd</a>, <a href="https://sthbrx.github.io/tag/cuda.html">cuda</a>, <a href="https://sthbrx.github.io/tag/gpu.html">gpu</a>, <a href="https://sthbrx.github.io/tag/minsky.html">minsky</a>, <a href="https://sthbrx.github.io/tag/s822lc-for-hpc.html">S822LC for hpc</a>, <a href="https://sthbrx.github.io/tag/debug.html">debug</a>, <a href="https://sthbrx.github.io/tag/virtualisation.html">virtualisation</a>, <a href="https://sthbrx.github.io/tag/dmesg.html">dmesg</a>, <a href="https://sthbrx.github.io/tag/printk.html">printk</a>, <a href="https://sthbrx.github.io/tag/boot.html">boot</a>, <a href="https://sthbrx.github.io/tag/early.html">early</a>, <a href="https://sthbrx.github.io/tag/error.html">error</a>, <a href="https://sthbrx.github.io/tag/centos.html">centos</a>, <a href="https://sthbrx.github.io/tag/centos7.html">centos7</a>, <a href="https://sthbrx.github.io/tag/p8.html">p8</a>, <a href="https://sthbrx.github.io/tag/bmc.html">bmc</a>, <a href="https://sthbrx.github.io/tag/rhel.html">RHEL</a>, <a href="https://sthbrx.github.io/tag/skiroot.html">skiroot</a>, <a href="https://sthbrx.github.io/tag/devmapper.html">devmapper</a>, <a href="https://sthbrx.github.io/tag/lvm.html">lvm</a>, <a href="https://sthbrx.github.io/tag/cgroups.html">cgroups</a>, <a href="https://sthbrx.github.io/tag/numa.html">numa</a>, <a href="https://sthbrx.github.io/tag/development.html">Development</a>, <a href="https://sthbrx.github.io/tag/netboot.html">netboot</a>, <a href="https://sthbrx.github.io/tag/pxe.html">pxe</a>, <a href="https://sthbrx.github.io/tag/education.html">Education</a>, <a href="https://sthbrx.github.io/tag/work-experience.html">work experience</a>, <a href="https://sthbrx.github.io/tag/asm.html">asm</a>, <a href="https://sthbrx.github.io/tag/vdso.html">vdso</a>, <a href="https://sthbrx.github.io/tag/snowpatch.html">snowpatch</a>, <a href="https://sthbrx.github.io/tag/tools.html">tools</a>, <a href="https://sthbrx.github.io/tag/intern.html">intern</a>, <a href="https://sthbrx.github.io/tag/srop.html">SROP</a>, <a href="https://sthbrx.github.io/tag/mitigation.html">mitigation</a>, <a href="https://sthbrx.github.io/tag/double.html">double</a>, <a href="https://sthbrx.github.io/tag/float.html">float</a>, <a href="https://sthbrx.github.io/tag/hex.html">hex</a>, <a href="https://sthbrx.github.io/tag/debugging.html">debugging</a>, <a href="https://sthbrx.github.io/tag/skiboot.html">skiboot</a>, <a href="https://sthbrx.github.io/tag/opal.html">OPAL</a>, <a href="https://sthbrx.github.io/tag/fsp.html">FSP</a>, <a href="https://sthbrx.github.io/tag/patches.html">patches</a>, <a href="https://sthbrx.github.io/tag/based16.html">based16</a>, <a href="https://sthbrx.github.io/tag/linux-gods.html">Linux Gods</a>, <a href="https://sthbrx.github.io/tag/ozlabs.html">Ozlabs</a>, <a href="https://sthbrx.github.io/tag/offtopic.html">offtopic</a>, <a href="https://sthbrx.github.io/tag/autoboot.html">autoboot</a>, <a href="https://sthbrx.github.io/tag/kexec.html">kexec</a>, <a href="https://sthbrx.github.io/tag/aufs.html">aufs</a>, <a href="https://sthbrx.github.io/tag/overlay.html">overlay</a>, <a href="https://sthbrx.github.io/tag/php.html">php</a>, <a href="https://sthbrx.github.io/tag/capi.html">capi</a> </section>
<section>
<h1><a href="https://sthbrx.github.io/authors.html">Authors</a></h1>
<ul id="authors_list">
<li><a href="https://sthbrx.github.io/author/alastair-dsilva.html">Alastair D'Silva</a></li>
<li><a href="https://sthbrx.github.io/author/andrew-donnellan.html">Andrew Donnellan</a></li>
<li><a href="https://sthbrx.github.io/author/anton-blanchard.html">Anton Blanchard</a></li>
<li><a href="https://sthbrx.github.io/author/benjamin-gray.html">Benjamin Gray</a></li>
<li><a href="https://sthbrx.github.io/author/callum-scarvell.html">Callum Scarvell</a></li>
<li><a href="https://sthbrx.github.io/author/cyril-bur.html">Cyril Bur</a></li>
<li><a href="https://sthbrx.github.io/author/daniel-axtens.html">Daniel Axtens</a></li>
<li><a href="https://sthbrx.github.io/author/daniel-black.html">Daniel Black</a></li>
<li><a href="https://sthbrx.github.io/author/joel-stanley.html">Joel Stanley</a></li>
<li><a href="https://sthbrx.github.io/author/nick-piggin.html">Nick Piggin</a></li>
<li><a href="https://sthbrx.github.io/author/rashmica-gupta.html">Rashmica Gupta</a></li>
<li><a href="https://sthbrx.github.io/author/rohan-mclure.html">Rohan McLure</a></li>
<li><a href="https://sthbrx.github.io/author/russell-currey.html">Russell Currey</a></li>
<li><a href="https://sthbrx.github.io/author/samuel-mendoza-jonas.html">Samuel Mendoza-Jonas</a></li>
<li><a href="https://sthbrx.github.io/author/suraj-jitindar-singh.html">Suraj Jitindar Singh</a></li>
</ul>
</section>
<section>
<h1>Social</h1>
<ul>
<li><a href="https://sthbrx.github.io/rss.xml" type="application/rss+xml" rel="alternate">RSS</a></li>
<li><a href="https://github.com/sthbrx/" target="_blank">GitHub</a></li>
<li><a href="https://lists.ozlabs.org/listinfo/linuxppc-dev" target="_blank">linuxppc mailing list</a></li>
<li><a href="https://lists.ozlabs.org/listinfo/skiboot" target="_blank">Skiboot mailing list</a></li>
</ul>
</section>
<section>
<h1>Blogroll</h1>
<ul>
<li><a href="http://ozlabs.org" target="_blank">OzLabs</a></li>
</ul>
</section>
<section>
<h1>Disclaimer</h1>
<div>
This blog represents the views of the individual authors, and doesn't necessarily represent IBM's positions, strategies or opinions. </div>
</section>
</aside> </div>
</div>
<footer role="contentinfo"><p>
Copyright © 2015–2023 OzLabs —
<span class="credit">Powered by <a href="http://getpelican.com">Pelican</a></span>
</p></footer>
</body>
</html>