Merge branch 'ceph:main' into master

debian/ceph-test.install

@@ -16,5 +16,6 @@ usr/bin/ceph_scratchtool
 usr/bin/ceph_scratchtoolpp
 usr/bin/ceph_test_*
 usr/bin/ceph-dedup-tool
+usr/bin/ceph-dedup-daemon
 usr/lib/ceph/ceph-monstore-update-crush.sh
 usr/share/java/libcephfs-test.jar

doc/cephfs/fs-volumes.rst

@@ -676,7 +676,7 @@ Cloning Snapshots
 
 Subvolumes can be created by cloning subvolume snapshots. Cloning is an
 asynchronous operation that copies data from a snapshot to a subvolume. Because
-cloning is an operation that involves bulk copying, it is inefficient for
+cloning is an operation that involves bulk copying, it is slow for
 very large data sets.
 
 .. note:: Removing a snapshot (source subvolume) fails when there are

doc/cephfs/mount-prerequisites.rst

@@ -2,11 +2,11 @@ Mount CephFS: Prerequisites
 ===========================
 
 You can use CephFS by mounting it to your local filesystem or by using
-`cephfs-shell`_. Mounting CephFS requires superuser privileges to trim
-dentries by issuing a remount of itself. CephFS can be mounted
-`using kernel`_ as well as `using FUSE`_. Both have their own
-advantages. Read the following section to understand more about both of
-these ways to mount CephFS.
+:ref:`cephfs-shell <cephfs-shell>`. Mounting CephFS requires superuser
+privileges to trim dentries by issuing a remount of itself. CephFS can be
+mounted `using kernel`_ as well as `using FUSE`_. Both have their own
+advantages. Read the following section to understand more about both of these
+ways to mount CephFS.
 
 For Windows CephFS mounts, please check the `ceph-dokan`_ page.
 
@@ -69,7 +69,7 @@ Ceph MON resides.
    individually, please check respective mount documents.
 
 .. _Client Authentication: ../client-auth
-.. _cephfs-shell: ../cephfs-shell
+.. _cephfs-shell: ..cephfs-shell
 .. _using kernel: ../mount-using-kernel-driver
 .. _using FUSE: ../mount-using-fuse
 .. _ceph-dokan: ../ceph-dokan

doc/man/8/cephfs-shell.rst

@@ -1,5 +1,7 @@
 :orphan:
 
+.. _cephfs-shell:
+
 ===================================================
 cephfs-shell -- Shell-like tool talking with CephFS
 ===================================================

doc/start/hardware-recommendations.rst

@@ -257,7 +257,7 @@ and Ceph Monitor / Manager purposes, a minimum capacity of 256GB is advised
 and at least 480GB is recommended. A drive model rated at 1+ DWPD (or the
 equivalent in TBW (TeraBytes Written) is suggested.  However, for a given write
 workload, a larger drive than technically required will provide more endurance
-because it effectively has greater overprovsioning. We stress that
+because it effectively has greater overprovisioning. We stress that
 enterprise-class drives are best for production use, as they feature power
 loss protection and increased durability compared to client (desktop) SKUs
 that are intended for much lighter and intermittent duty cycles.

doc/start/os-recommendations.rst

@@ -49,9 +49,9 @@ non-Linux systems but these are not supported by the core Ceph effort.
 +===============+===============+=================+==================+==================+
 | Centos 7      |               |                 |         A        |      B           |
 +---------------+---------------+-----------------+------------------+------------------+
-| Centos 8      |    A          |     A           |         A        |      A           |
+| Centos 8      |               |                 |                  |                  |
 +---------------+---------------+-----------------+------------------+------------------+
-| Centos 9      |    A          |                 |                  |                  |
+| Centos 9      |    A          |     A :sup:`1`  |                  |                  |
 +---------------+---------------+-----------------+------------------+------------------+
 | Debian 10     |    C          |                 |         C        |      C           |
 +---------------+---------------+-----------------+------------------+------------------+
@@ -71,6 +71,7 @@ non-Linux systems but these are not supported by the core Ceph effort.
 - **A**: Ceph provides packages and has done comprehensive tests on the software in them.
 - **B**: Ceph provides packages and has done basic tests on the software in them.
 - **C**: Ceph provides packages only. No tests have been done on these releases.
+- **1**: Testing has been done on Centos 9 starting on version 17.2.8 for Quincy.
 
 .. note::
    **For Centos 7 Users** 

qa/config/bluestore.yaml

@@ -0,0 +1,23 @@
+overrides:
+  ceph:
+    fs: xfs
+    conf:
+      osd:
+        osd objectstore: bluestore
+        bluestore block size: 96636764160
+        debug bluestore: 20
+        debug bluefs: 20
+        debug rocksdb: 10
+        bluestore compression mode: aggressive
+        bluestore fsck on mount: true
+        bluestore compression algorithm: snappy
+        # lower the full ratios since we can fill up a 100gb osd so quickly
+        mon osd full ratio: .9
+        mon osd backfillfull_ratio: .85
+        mon osd nearfull ratio: .8
+        osd failsafe full ratio: .95
+        bluestore rocksdb cf: false
+        log to stderr: true
+        err to stderr: true
+        log flush on exit: true
+        log to file: false

qa/config/seastore.yaml

@@ -0,0 +1,6 @@
+overrides:
+  ceph:
+    fs: xfs
+    conf:
+      osd:
+        osd objectstore: seastore

qa/suites/crimson-rados-experimental/seastore/basic/objectstore/seastore.yaml

@@ -0,0 +1 @@
+.qa/config/seastore.yaml

qa/suites/crimson-rados/basic/objectstore/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/crimson-rados/basic/objectstore/bluestore.yaml

@@ -0,0 +1 @@
+.qa/config/bluestore.yaml

qa/suites/crimson-rados/basic/objectstore/seastore.yaml

@@ -0,0 +1 @@
+.qa/config/seastore.yaml

qa/suites/crimson-rados/perf/objectstore/bluestore.yaml

@@ -0,0 +1 @@
+.qa/config/bluestore.yaml

qa/suites/crimson-rados/perf/objectstore/seastore.yaml

@@ -0,0 +1 @@
+.qa/config/seastore.yaml

qa/suites/crimson-rados/rbd/objectstore/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/crimson-rados/rbd/objectstore/bluestore.yaml

@@ -0,0 +1 @@
+.qa/config/bluestore.yaml

qa/suites/crimson-rados/rbd/objectstore/seastore.yaml

@@ -0,0 +1 @@
+.qa/config/seastore.yaml

qa/suites/crimson-rados/thrash/objectstore/bluestore.yaml

@@ -0,0 +1 @@
+.qa/config/bluestore.yaml

qa/suites/crimson-rados/thrash_simple/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/crimson-rados/thrash_simple/0-size-min-size-overrides/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/crimson-rados/thrash_simple/0-size-min-size-overrides/2-size-2-min-size.yaml.disabled

@@ -0,0 +1 @@
+.qa/overrides/2-size-2-min-size.yaml

qa/suites/crimson-rados/thrash_simple/0-size-min-size-overrides/3-size-2-min-size.yaml

@@ -0,0 +1 @@
+.qa/overrides/3-size-2-min-size.yaml

qa/suites/crimson-rados/thrash_simple/1-pg-log-overrides/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/crimson-rados/thrash_simple/1-pg-log-overrides/short_pg_log.yaml.disabled

@@ -0,0 +1 @@
+.qa/overrides/short_pg_log.yaml

qa/suites/crimson-rados/thrash_simple/2-recovery-overrides/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/crimson-rados/thrash_simple/2-recovery-overrides/more-active-recovery.yaml.disabled

@@ -0,0 +1 @@
+.qa/overrides/more-active-recovery.yaml

qa/suites/crimson-rados/thrash_simple/2-recovery-overrides/more-async-partial-recovery.yaml.disabled

@@ -0,0 +1,6 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        osd_async_recovery_min_cost: 1
+        osd_object_clean_region_max_num_intervals: 1000

qa/suites/crimson-rados/thrash_simple/2-recovery-overrides/more-async-recovery.yaml.disabled

@@ -0,0 +1,5 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        osd_async_recovery_min_cost: 1

qa/suites/crimson-rados/thrash_simple/2-recovery-overrides/more-partial-recovery.yaml.disabled

@@ -0,0 +1,5 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        osd_object_clean_region_max_num_intervals: 1000

qa/suites/crimson-rados/thrash_simple/clusters/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/crimson-rados/thrash_simple/clusters/fixed-2.yaml

@@ -0,0 +1,24 @@
+roles:
+- [mon.a, osd.0, osd.1, client.0, node-exporter.a]
+- [mgr.x, osd.2, osd.3, client.1, prometheus.a, node-exporter.b]
+overrides:
+  ceph:
+    conf:
+      osd:
+        osd shutdown pgref assert: true
+        crimson alien thread cpu cores: 6-7
+      osd.0:
+        crimson seastar cpu cores: 0-2
+      osd.1:
+        crimson seastar cpu cores: 3-5
+      osd.2:
+        crimson seastar cpu cores: 0-2
+      osd.3:
+        crimson seastar cpu cores: 3-5
+      global:
+        ms cluster mode: crc
+        ms service mode: crc
+        ms client mode: crc
+        ms mon service mode: crc
+        ms mon cluster mode: crc
+        ms mon client mode: crc

qa/suites/crimson-rados/thrash_simple/clusters/openstack.yaml.disabled

@@ -0,0 +1,4 @@
+openstack:
+  - volumes: # attached to each instance
+      count: 4
+      size: 10 # GB

qa/suites/crimson-rados/thrash_simple/crimson-supported-all-distro

@@ -0,0 +1 @@
+.qa/distros/crimson-supported-all-distro/

qa/suites/crimson-rados/thrash_simple/crimson_qa_overrides.yaml

@@ -0,0 +1 @@
+.qa/config/crimson_qa_overrides.yaml

qa/suites/crimson-rados/thrash_simple/deploy/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/crimson-rados/thrash_simple/deploy/ceph.yaml

@@ -0,0 +1,11 @@
+overrides:
+  install:
+    ceph:
+      flavor: crimson
+tasks:
+- install:
+- ceph:
+    conf:
+      osd:
+        debug monc: 20
+    flavor: crimson

qa/suites/crimson-rados/thrash_simple/deploy/cephadm.yaml.disabled

@@ -0,0 +1,16 @@
+# no need to verify os + flavor + sha1
+verify_ceph_hash: false
+tasks:
+- cephadm:
+    conf:
+      mgr:
+        debug ms: 1
+        debug mgr: 20
+        debug osd: 10
+- cephadm.shell:
+    mon.a:
+      - ceph orch status
+      - ceph orch ps
+      - ceph orch ls
+      - ceph orch host ls
+      - ceph orch device ls

qa/suites/crimson-rados/thrash_simple/objectstore/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/crimson-rados/thrash_simple/objectstore/bluestore.yaml

@@ -0,0 +1 @@
+.qa/config/bluestore.yaml

qa/suites/crimson-rados/thrash_simple/objectstore/seastore.yaml

@@ -0,0 +1 @@
+.qa/config/seastore.yaml

qa/suites/crimson-rados/thrash_simple/thrashers/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/crimson-rados/thrash_simple/thrashosds-health.yaml

@@ -0,0 +1 @@
+.qa/tasks/thrashosds-health.yaml

qa/suites/crimson-rados/thrash_simple/workloads/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/crimson-rados/thrash_simple/workloads/admin_socket_objecter_requests.yaml

@@ -0,0 +1,13 @@
+overrides:
+  ceph:
+    conf:
+      client.0:
+        admin socket: /var/run/ceph/ceph-$name.asok
+tasks:
+- radosbench:
+    clients: [client.0]
+    time: 150
+- admin_socket:
+    client.0:
+      objecter_requests:
+        test: "http://git.ceph.com/?p={repo};a=blob_plain;f=src/test/admin_socket/objecter_requests;hb={branch}"

qa/suites/crimson-rados/thrash_simple/workloads/pool-snaps-few-objects.yaml

@@ -0,0 +1,20 @@
+overrides:
+  conf:
+    osd:
+      osd deep scrub update digest min age: 0
+tasks:
+- rados:
+    clients: [client.0]
+    ops: 4000
+    objects: 50
+    pool_snaps: true
+    op_weights:
+      read: 100
+      write: 100
+      delete: 50
+      snap_create: 50
+      snap_remove: 50
+      rollback: 0
+      # TODO: CEPH_OSD_OP_COPY_FROM
+      copy_from: 0
+