All test clusters are running:
- k8s
v1.23.7-rancher1-1
- rook 1.10.1
- ceph 17.2.3
Two baremetal clusters are being used for testing.
- ruka is the "primary" cluster with the master zone named
zone-a
. - kueyen is the "secondary" cluster with a zone named
zone-b
.
The ruka configuration is based on:
The kueyen configuration is based on:
The changes are minor. The realm-a secret key, increasing the number of replicas to 3 to prevent operator complaints, and setting endpoints to use an ingress resources.
The test procedure has been to leave the "primary" ceph cluster with the master zone intact between test runs with only the "secondary" cluster's zone manually deleted from the zonegroup. The secondary ceph cluster is completely destroyed with OSDs wiped between test runs. However, the secondary cluster k8s is not reprovisoned between test runs.
Note that console output has been edited to redact secrets, bash prompt, and domain name.
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin zonegroup get
{
"id": "003fe63e-b238-400f-aa5e-ecf84b221632",
"name": "zonegroup-a",
"api_name": "zonegroup-a",
"is_master": "true",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"zones": [
{
"id": "a8919925-fa14-41e4-a9e1-8fdbd56dab79",
"name": "zone-b",
"endpoints": [
"https://s3test.kueyen.ls.example.com",
"http://10.43.108.148:80"
],
"log_meta": "false",
"log_data": "true",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
},
{
"id": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"name": "zone-a",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"log_meta": "false",
"log_data": "true",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
],
"default_placement": "default-placement",
"realm_id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"sync_policy": {
"groups": []
}
}
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin zonegroup remove --rgw-zonegroup=zonegroup-a --rgw-zone=zone-b
{
"id": "003fe63e-b238-400f-aa5e-ecf84b221632",
"name": "zonegroup-a",
"api_name": "zonegroup-a",
"is_master": "true",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"zones": [
{
"id": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"name": "zone-a",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
],
"default_placement": "default-placement",
"realm_id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"sync_policy": {
"groups": []
}
}
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin period update --commit
{
"id": "45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53",
"epoch": 19,
"predecessor_uuid": "0161d1d2-3f7d-446e-bda6-3ec007e5892a",
"sync_status": [],
"period_map": {
"id": "45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53",
"zonegroups": [
{
"id": "003fe63e-b238-400f-aa5e-ecf84b221632",
"name": "zonegroup-a",
"api_name": "zonegroup-a",
"is_master": "true",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"zones": [
{
"id": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"name": "zone-a",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
],
"default_placement": "default-placement",
"realm_id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"sync_policy": {
"groups": []
}
}
],
"short_zone_ids": [
{
"key": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"val": 3237871277
}
]
},
"master_zonegroup": "003fe63e-b238-400f-aa5e-ecf84b221632",
"master_zone": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"period_config": {
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"user_ratelimit": {
"max_read_ops": 0,
"max_write_ops": 0,
"max_read_bytes": 0,
"max_write_bytes": 0,
"enabled": false
},
"bucket_ratelimit": {
"max_read_ops": 0,
"max_write_ops": 0,
"max_read_bytes": 0,
"max_write_bytes": 0,
"enabled": false
},
"anonymous_ratelimit": {
"max_read_ops": 0,
"max_write_ops": 0,
"max_read_bytes": 0,
"max_write_bytes": 0,
"enabled": false
}
},
"realm_id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"realm_name": "realm-a",
"realm_epoch": 2
}
[rook@rook-ceph-tools-f7f46787d-phght /]$ ceph status
cluster:
id: df44afcb-2a89-41d8-b77f-7568e880966d
health: HEALTH_OK
services:
mon: 3 daemons, quorum a,b,c (age 7m)
mgr: a(active, since 6m), standbys: b
osd: 12 osds: 12 up (since 111s), 12 in (since 5m)
data:
pools: 1 pools, 1 pgs
objects: 2 objects, 449 KiB
usage: 1.0 GiB used, 29 TiB / 29 TiB avail
pgs: 1 active+clean
[rook@rook-ceph-tools-f7f46787d-phght /]$ rados df
POOL_NAME USED OBJECTS CLONES COPIES MISSING_ON_PRIMARY UNFOUND DEGRADED RD_OPS RD WR_OPS WR USED COMPR UNDER COMPR
.mgr 449 KiB 2 0 6 0 0 0 97 83 KiB 114 1.3 MiB 0 B 0 B
total_objects 2
total_used 1.0 GiB
total_avail 29 TiB
total_space 29 TiB
$ k apply -f realm-a-keys.yaml
secret/realm-a-keys created
No reaction from operator.
$ k apply -f multisite-ingress.yaml
ingress.networking.k8s.io/rook-ceph-rgw-zone-b-multisite-store-ingress created
No reaction from operator.
$ cat realm-a.yaml
---
apiVersion: ceph.rook.io/v1
kind: CephObjectRealm
metadata:
name: realm-a
namespace: rook-ceph
spec:
pull:
endpoint: "https://s3test.ruka.dev.example.com"
$ k apply -f realm-a.yaml
cephobjectrealm.ceph.rook.io/realm-a created
secondary operator log:
2022-09-27 16:26:15.823397 I | ceph-spec: parsing mon endpoints: a=10.43.58.160:6789,b=10.43.147.40:6789,c=10.43.190.167:6789
secondary ceph:
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin realm list
{
"default_info": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"realms": [
"realm-a"
]
}
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin realm get
{
"id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"name": "realm-a",
"current_period": "45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53",
"epoch": 2
}
primary ceph:
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin realm get
{
"id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"name": "realm-a",
"current_period": "45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53",
"epoch": 2
}
$ cat zonegroup-a.yaml
---
apiVersion: ceph.rook.io/v1
kind: CephObjectZoneGroup
metadata:
name: zonegroup-a
namespace: rook-ceph
spec:
realm: realm-a
$ k apply -f zonegroup-a.yaml
cephobjectzonegroup.ceph.rook.io/zonegroup-a created
secondary operator log:
2022-09-27 16:30:35.657065 I | ceph-spec: parsing mon endpoints: a=10.43.58.160:6789,b=10.43.147.40:6789,c=10.43.190.167:6789
2022-09-27 16:30:35.666785 I | ceph-object-zonegroup-controller: CephObjectRealm "realm-a" found for CephObjectZoneGroup "zonegroup-a"
2022-09-27 16:30:36.198784 I | ceph-object-zonegroup-controller: Realm "realm-a" found in Ceph cluster to create ceph zone group "zonegroup-a"
2022-09-27 16:30:36.198819 I | ceph-object-zonegroup-controller: creating object zone group "zonegroup-a" in realm "realm-a"
secondary ceph:
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin zonegroup list
{
"default_info": "003fe63e-b238-400f-aa5e-ecf84b221632",
"zonegroups": [
"zonegroup-a"
]
}
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin zonegroup get
{
"id": "003fe63e-b238-400f-aa5e-ecf84b221632",
"name": "zonegroup-a",
"api_name": "zonegroup-a",
"is_master": "true",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"zones": [
{
"id": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"name": "zone-a",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
],
"default_placement": "default-placement",
"realm_id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"sync_policy": {
"groups": []
}
}
primary ceph:
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin zonegroup get
{
"id": "003fe63e-b238-400f-aa5e-ecf84b221632",
"name": "zonegroup-a",
"api_name": "zonegroup-a",
"is_master": "true",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"zones": [
{
"id": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"name": "zone-a",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
],
"default_placement": "default-placement",
"realm_id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"sync_policy": {
"groups": []
}
}
$ cat zone-b.yaml
---
apiVersion: ceph.rook.io/v1
kind: CephObjectZone
metadata:
name: zone-b
namespace: rook-ceph
spec:
zoneGroup: zonegroup-a
metadataPool:
failureDomain: host
replicated:
size: 3
requireSafeReplicaSize: false
dataPool:
failureDomain: host
replicated:
size: 3
requireSafeReplicaSize: false
customEndpoints:
- "https://s3test.kueyen.ls.example.com"
$ k apply -f zone-b.yaml
cephobjectzone.ceph.rook.io/zone-b created
secondary operator log:
2022-09-27 16:33:42.686264 I | ceph-spec: parsing mon endpoints: a=10.43.58.160:6789,b=10.43.147.40:6789,c=10.43.190.167:6789
2022-09-27 16:33:45.193779 I | ceph-object-zone-controller: Zone group "zonegroup-a" found in Ceph cluster to create ceph zone "zone-b"
2022-09-27 16:33:45.193822 I | ceph-object-zone-controller: creating object zone "zone-b" in zonegroup "zonegroup-a" in realm "realm-a"
2022-09-27 16:33:46.876591 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2
2022-09-27 16:33:54.465531 I | cephclient: reconciling replicated pool zone-b.rgw.control succeeded
2022-09-27 16:34:07.656281 I | cephclient: reconciling replicated pool zone-b.rgw.meta succeeded
2022-09-27 16:34:21.172438 I | cephclient: reconciling replicated pool zone-b.rgw.log succeeded
2022-09-27 16:34:33.373171 I | cephclient: reconciling replicated pool zone-b.rgw.buckets.index succeeded
2022-09-27 16:34:43.535927 I | cephclient: reconciling replicated pool zone-b.rgw.buckets.non-ec succeeded
2022-09-27 16:34:51.826012 I | cephclient: reconciling replicated pool .rgw.root succeeded
2022-09-27 16:35:02.475407 I | cephclient: reconciling replicated pool zone-b.rgw.buckets.data succeeded
secondary ceph:
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin zonegroup get
{
"id": "003fe63e-b238-400f-aa5e-ecf84b221632",
"name": "zonegroup-a",
"api_name": "zonegroup-a",
"is_master": "true",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"zones": [
{
"id": "2f90434c-2764-48ab-bba3-bcffa042dff1",
"name": "zone-b",
"endpoints": [
"https://s3test.kueyen.ls.example.com"
],
"log_meta": "false",
"log_data": "true",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
},
{
"id": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"name": "zone-a",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"log_meta": "false",
"log_data": "true",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
],
"default_placement": "default-placement",
"realm_id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"sync_policy": {
"groups": []
}
}
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin zone list
{
"default_info": "2f90434c-2764-48ab-bba3-bcffa042dff1",
"zones": [
"zone-b"
]
}
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin zone get
{
"id": "2f90434c-2764-48ab-bba3-bcffa042dff1",
"name": "zone-b",
"domain_root": "zone-b.rgw.meta:root",
"control_pool": "zone-b.rgw.control",
"gc_pool": "zone-b.rgw.log:gc",
"lc_pool": "zone-b.rgw.log:lc",
"log_pool": "zone-b.rgw.log",
"intent_log_pool": "zone-b.rgw.log:intent",
"usage_log_pool": "zone-b.rgw.log:usage",
"roles_pool": "zone-b.rgw.meta:roles",
"reshard_pool": "zone-b.rgw.log:reshard",
"user_keys_pool": "zone-b.rgw.meta:users.keys",
"user_email_pool": "zone-b.rgw.meta:users.email",
"user_swift_pool": "zone-b.rgw.meta:users.swift",
"user_uid_pool": "zone-b.rgw.meta:users.uid",
"otp_pool": "zone-b.rgw.otp",
"system_key": {
"access_key": "<foo>",
"secret_key": "<bar>"
},
"placement_pools": [
{
"key": "default-placement",
"val": {
"index_pool": "zone-b.rgw.buckets.index",
"storage_classes": {
"STANDARD": {
"data_pool": "zone-b.rgw.buckets.data"
}
},
"data_extra_pool": "zone-b.rgw.buckets.non-ec",
"index_type": 0
}
}
],
"realm_id": "",
"notif_pool": "zone-b.rgw.log:notif"
}
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin period get-current
{
"current_period": "45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53"
}
primary ceph:
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin zonegroup get
{
"id": "003fe63e-b238-400f-aa5e-ecf84b221632",
"name": "zonegroup-a",
"api_name": "zonegroup-a",
"is_master": "true",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"zones": [
{
"id": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"name": "zone-a",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
],
"default_placement": "default-placement",
"realm_id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"sync_policy": {
"groups": []
}
}
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin zone list
{
"default_info": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"zones": [
"lfa",
"zone-a"
]
}
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin period get-current
{
"current_period": "45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53"
}
$ cat zone-b-multisite-store.yaml
---
apiVersion: ceph.rook.io/v1
kind: CephObjectStore
metadata:
name: zone-b-multisite-store
namespace: rook-ceph
spec:
gateway:
port: 80
# securePort: 443
instances: 1
zone:
name: zone-b
$ k apply -f zone-b-multisite-store.yaml
cephobjectstore.ceph.rook.io/zone-b-multisite-store created
secondary operator:
2022-09-27 16:42:04.925549 I | rook-ceph-webhook: validate create cephobjectstore &{{CephObjectStore ceph.rook.io/v1} {zone-b-multisite-store rook-ceph 3672726d-bc30-49ac-b7a7-eaedf24a6dc1 1 2022-09-27 16:42:04 +0000 UTC <nil> <nil> map[] map[kubectl.kubernetes.io/last-applied-configuration:{"apiVersion":"ceph.rook.io/v1","kind":"CephObjectStore","metadata":{"annotations":{},"name":"zone-b-multisite-store","namespace":"rook-ceph"},"spec":{"gateway":{"instances":1,"port":80},"zone":{"name":"zone-b"}}}
] [] [] [{kubectl-client-side-apply Update ceph.rook.io/v1 2022-09-27 16:42:04 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{}}},"f:spec":{".":{},"f:gateway":{".":{},"f:instances":{},"f:port":{}},"f:zone":{".":{},"f:name":{}}}} }]} {{ {0 0 false 0 <nil>} {0 0 } map[] false {false [] <nil>} {{false nil }} {<nil> <nil> <nil>}} { {0 0 false 0 <nil>} {0 0 } map[] false {false [] <nil>} {{false nil }} {<nil> <nil> <nil>}} false {80 0 1 {nil nil nil [] []} map[] map[] {map[] map[]} [] <nil> <nil>} {zone-b} {{false nil } <nil> <nil> <nil>} <nil>} <nil>}
2022-09-27 16:42:04.938033 I | ceph-spec: adding finalizer "cephobjectstore.ceph.rook.io" on "zone-b-multisite-store"
2022-09-27 16:42:04.953183 I | rook-ceph-webhook: validate update cephobjectstore
2022-09-27 16:42:05.070855 I | ceph-spec: parsing mon endpoints: a=10.43.58.160:6789,b=10.43.147.40:6789,c=10.43.190.167:6789
2022-09-27 16:42:05.070957 I | ceph-spec: detecting the ceph image version for image quay.io/ceph/ceph:v17.2.3...
2022-09-27 16:42:06.393410 I | clusterdisruption-controller: all PGs are active+clean. Restoring default OSD pdb settings
2022-09-27 16:42:06.393446 I | clusterdisruption-controller: creating the default pdb "rook-ceph-osd" with maxUnavailable=1 for all osd
2022-09-27 16:42:13.871697 I | ceph-spec: detected ceph image version: "17.2.3-0 quincy"
2022-09-27 16:42:15.330339 I | ceph-object-controller: reconciling object store deployments
2022-09-27 16:42:15.944758 I | ceph-object-controller: Zone "zone-b" found in Ceph cluster will include object store "zone-b-multisite-store"
2022-09-27 16:42:15.962446 I | ceph-object-controller: ceph object store gateway service running at 10.43.225.199
2022-09-27 16:42:15.962486 I | ceph-object-controller: setting multisite settings for object store "zone-b-multisite-store"
2022-09-27 16:42:15.962496 I | ceph-object-controller: multisite configuration for object-store zone-b-multisite-store is complete
2022-09-27 16:42:15.962512 I | ceph-object-controller: creating object store "zone-b-multisite-store" in namespace "rook-ceph"
2022-09-27 16:42:15.962527 I | cephclient: getting or creating ceph auth key "client.rgw.zone.b.multisite.store.a"
2022-09-27 16:42:17.388469 I | ceph-object-controller: setting rgw config flags
2022-09-27 16:42:17.388518 I | op-config: setting "client.rgw.zone.b.multisite.store.a"="rgw_zonegroup"="zonegroup-a" option to the mon configuration database
2022-09-27 16:42:18.522936 I | op-config: successfully set "client.rgw.zone.b.multisite.store.a"="rgw_zonegroup"="zonegroup-a" option to the mon configuration database
2022-09-27 16:42:18.523000 I | op-config: setting "client.rgw.zone.b.multisite.store.a"="rgw_log_nonexistent_bucket"="true" option to the mon configuration database
2022-09-27 16:42:20.272091 I | op-config: successfully set "client.rgw.zone.b.multisite.store.a"="rgw_log_nonexistent_bucket"="true" option to the mon configuration database
2022-09-27 16:42:20.272144 I | op-config: setting "client.rgw.zone.b.multisite.store.a"="rgw_log_object_name_utc"="true" option to the mon configuration database
2022-09-27 16:42:23.272386 I | op-config: successfully set "client.rgw.zone.b.multisite.store.a"="rgw_log_object_name_utc"="true" option to the mon configuration database
2022-09-27 16:42:23.272432 I | op-config: setting "client.rgw.zone.b.multisite.store.a"="rgw_enable_usage_log"="true" option to the mon configuration database
2022-09-27 16:42:25.679916 I | op-config: successfully set "client.rgw.zone.b.multisite.store.a"="rgw_enable_usage_log"="true" option to the mon configuration database
2022-09-27 16:42:25.679960 I | op-config: setting "client.rgw.zone.b.multisite.store.a"="rgw_zone"="zone-b" option to the mon configuration database
2022-09-27 16:42:26.823842 I | op-config: successfully set "client.rgw.zone.b.multisite.store.a"="rgw_zone"="zone-b" option to the mon configuration database
2022-09-27 16:42:26.824240 I | ceph-object-controller: object store "zone-b-multisite-store" deployment "rook-ceph-rgw-zone-b-multisite-store-a" created
2022-09-27 16:42:26.928863 I | ceph-object-controller: enabling rgw dashboard
2022-09-27 16:42:27.090093 E | ceph-crashcollector-controller: node reconcile failed on op "unchanged": Operation cannot be fulfilled on deployments.apps "rook-ceph-crashcollector-kueyen02": the object has been modified; please apply your changes to the latest version and try again
2022-09-27 16:42:27.174714 E | ceph-crashcollector-controller: node reconcile failed on op "unchanged": Operation cannot be fulfilled on deployments.apps "rook-ceph-crashcollector-kueyen02": the object has been modified; please apply your changes to the latest version and try again
2022-09-27 16:42:31.688588 I | ceph-object-controller: setting the dashboard api secret key
2022-09-27 16:42:31.688751 I | ceph-object-controller: created object store "zone-b-multisite-store" in namespace "rook-ceph"
2022-09-27 16:42:32.894092 I | ceph-object-controller: starting rgw health checker for CephObjectStore "rook-ceph/zone-b-multisite-store"
2022-09-27 16:42:34.274641 I | ceph-object-controller: done setting the dashboard api secret key
secondary ceph:
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin zonegroup get
{
"id": "003fe63e-b238-400f-aa5e-ecf84b221632",
"name": "zonegroup-a",
"api_name": "zonegroup-a",
"is_master": "true",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"zones": [
{
"id": "2f90434c-2764-48ab-bba3-bcffa042dff1",
"name": "zone-b",
"endpoints": [
"https://s3test.kueyen.ls.example.com"
],
"log_meta": "false",
"log_data": "true",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
},
{
"id": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"name": "zone-a",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"log_meta": "false",
"log_data": "true",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
],
"default_placement": "default-placement",
"realm_id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"sync_policy": {
"groups": []
}
}
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin zone list
{
"default_info": "2f90434c-2764-48ab-bba3-bcffa042dff1",
"zones": [
"zone-b"
]
}
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin zone get
{
"id": "2f90434c-2764-48ab-bba3-bcffa042dff1",
"name": "zone-b",
"domain_root": "zone-b.rgw.meta:root",
"control_pool": "zone-b.rgw.control",
"gc_pool": "zone-b.rgw.log:gc",
"lc_pool": "zone-b.rgw.log:lc",
"log_pool": "zone-b.rgw.log",
"intent_log_pool": "zone-b.rgw.log:intent",
"usage_log_pool": "zone-b.rgw.log:usage",
"roles_pool": "zone-b.rgw.meta:roles",
"reshard_pool": "zone-b.rgw.log:reshard",
"user_keys_pool": "zone-b.rgw.meta:users.keys",
"user_email_pool": "zone-b.rgw.meta:users.email",
"user_swift_pool": "zone-b.rgw.meta:users.swift",
"user_uid_pool": "zone-b.rgw.meta:users.uid",
"otp_pool": "zone-b.rgw.otp",
"system_key": {
"access_key": "<foo>",
"secret_key": "<bar>"
},
"placement_pools": [
{
"key": "default-placement",
"val": {
"index_pool": "zone-b.rgw.buckets.index",
"storage_classes": {
"STANDARD": {
"data_pool": "zone-b.rgw.buckets.data"
}
},
"data_extra_pool": "zone-b.rgw.buckets.non-ec",
"index_type": 0
}
}
],
"realm_id": "",
"notif_pool": "zone-b.rgw.log:notif"
}
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin period get-current
{
"current_period": "45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53"
}
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin sync status
2022-09-27T16:48:31.698+0000 7fb232d0d500 0 period (45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53 does not have zone 2f90434c-2764-48ab-bba3-bcffa042dff1 configured
realm 3fc7538d-9a0f-4385-9867-e16200927e9c (realm-a)
zonegroup 003fe63e-b238-400f-aa5e-ecf84b221632 (zonegroup-a)
zone 2f90434c-2764-48ab-bba3-bcffa042dff1 (zone-b)
metadata sync syncing
full sync: 0/64 shards
incremental sync: 64/64 shards
metadata is caught up with master
data sync source: e5f5f214-5f18-4011-a460-aa047000ac7c (zone-a)
syncing
full sync: 0/128 shards
incremental sync: 128/128 shards
data is caught up with source
$ curl -I https://s3test.kueyen.ls.example.com/
HTTP/2 200
date: Tue, 27 Sep 2022 16:50:18 GMT
content-type: application/xml
x-amz-request-id: tx00000bf735e48609b2c06-00633329ca-41ec-zone-b
strict-transport-security: max-age=15724800; includeSubDomains
primary ceph:
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin zonegroup get
{
"id": "003fe63e-b238-400f-aa5e-ecf84b221632",
"name": "zonegroup-a",
"api_name": "zonegroup-a",
"is_master": "true",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"zones": [
{
"id": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"name": "zone-a",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
],
"default_placement": "default-placement",
"realm_id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"sync_policy": {
"groups": []
}
}
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin zone list
{
"default_info": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"zones": [
"lfa",
"zone-a"
]
}
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin period get-current
{
"current_period": "45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53"
}
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin sync status
realm 3fc7538d-9a0f-4385-9867-e16200927e9c (realm-a)
zonegroup 003fe63e-b238-400f-aa5e-ecf84b221632 (zonegroup-a)
zone e5f5f214-5f18-4011-a460-aa047000ac7c (zone-a)
metadata sync no sync (zone is master)
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin period update --commit
2022-09-27T16:52:59.500+0000 7f1b9042b500 0 period (45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53 does not have zone 2f90434c-2764-48ab-bba3-bcffa042dff1 configured
Sending period to new master zone e5f5f214-5f18-4011-a460-aa047000ac7c
{
"id": "45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53",
"epoch": 20,
"predecessor_uuid": "0161d1d2-3f7d-446e-bda6-3ec007e5892a",
"sync_status": [],
"period_map": {
"id": "45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53",
"zonegroups": [
{
"id": "003fe63e-b238-400f-aa5e-ecf84b221632",
"name": "zonegroup-a",
"api_name": "zonegroup-a",
"is_master": "true",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"zones": [
{
"id": "2f90434c-2764-48ab-bba3-bcffa042dff1",
"name": "zone-b",
"endpoints": [
"https://s3test.kueyen.ls.example.com"
],
"log_meta": "false",
"log_data": "true",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
},
{
"id": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"name": "zone-a",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"log_meta": "false",
"log_data": "true",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
],
"default_placement": "default-placement",
"realm_id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"sync_policy": {
"groups": []
}
}
],
"short_zone_ids": [
{
"key": "2f90434c-2764-48ab-bba3-bcffa042dff1",
"val": 96351969
},
{
"key": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"val": 3237871277
}
]
},
"master_zonegroup": "003fe63e-b238-400f-aa5e-ecf84b221632",
"master_zone": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"period_config": {
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"user_ratelimit": {
"max_read_ops": 0,
"max_write_ops": 0,
"max_read_bytes": 0,
"max_write_bytes": 0,
"enabled": false
},
"bucket_ratelimit": {
"max_read_ops": 0,
"max_write_ops": 0,
"max_read_bytes": 0,
"max_write_bytes": 0,
"enabled": false
},
"anonymous_ratelimit": {
"max_read_ops": 0,
"max_write_ops": 0,
"max_read_bytes": 0,
"max_write_bytes": 0,
"enabled": false
}
},
"realm_id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"realm_name": "realm-a",
"realm_epoch": 2
}
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin zonegroup get
{
"id": "003fe63e-b238-400f-aa5e-ecf84b221632",
"name": "zonegroup-a",
"api_name": "zonegroup-a",
"is_master": "true",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"zones": [
{
"id": "2f90434c-2764-48ab-bba3-bcffa042dff1",
"name": "zone-b",
"endpoints": [
"https://s3test.kueyen.ls.example.com"
],
"log_meta": "false",
"log_data": "true",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
},
{
"id": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"name": "zone-a",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"log_meta": "false",
"log_data": "true",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
],
"default_placement": "default-placement",
"realm_id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"sync_policy": {
"groups": []
}
}
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin period get-current
{
"current_period": "45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53"
}
[rook@rook-ceph-tools-f7f46787d-phght /]$ radosgw-admin sync status
realm 3fc7538d-9a0f-4385-9867-e16200927e9c (realm-a)
zonegroup 003fe63e-b238-400f-aa5e-ecf84b221632 (zonegroup-a)
zone 2f90434c-2764-48ab-bba3-bcffa042dff1 (zone-b)
metadata sync syncing
full sync: 0/64 shards
incremental sync: 64/64 shards
metadata is caught up with master
data sync source: e5f5f214-5f18-4011-a460-aa047000ac7c (zone-a)
syncing
full sync: 0/128 shards
incremental sync: 128/128 shards
data is behind on 1 shards
behind shards: [85]
oldest incremental change not applied: 2022-09-27T16:53:23.319216+0000 [85]
primary ceph:
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin zonegroup get
{
"id": "003fe63e-b238-400f-aa5e-ecf84b221632",
"name": "zonegroup-a",
"api_name": "zonegroup-a",
"is_master": "true",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"zones": [
{
"id": "2f90434c-2764-48ab-bba3-bcffa042dff1",
"name": "zone-b",
"endpoints": [
"https://s3test.kueyen.ls.example.com"
],
"log_meta": "false",
"log_data": "true",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
},
{
"id": "e5f5f214-5f18-4011-a460-aa047000ac7c",
"name": "zone-a",
"endpoints": [
"https://s3test.ruka.dev.example.com"
],
"log_meta": "false",
"log_data": "true",
"bucket_index_max_shards": 11,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
],
"default_placement": "default-placement",
"realm_id": "3fc7538d-9a0f-4385-9867-e16200927e9c",
"sync_policy": {
"groups": []
}
}
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin period get-current
{
"current_period": "45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53"
}
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin period list
{
"periods": [
"0161d1d2-3f7d-446e-bda6-3ec007e5892a",
"10817ed8-765d-4198-9382-089c06a35e0c:staging",
"16a1043f-798c-448a-ae02-7441390fef03:staging",
"179c319d-c430-4372-b751-f7f8b658a15e",
"1eca6078-dffe-4a95-a6b1-6013458f9332",
"3fc7538d-9a0f-4385-9867-e16200927e9c:staging",
"45a13413-7e1f-4bb8-aeb1-f3ce1a56fa53",
"65381de4-643e-44e4-aa10-304e53bbc514",
"75b1b923-198a-49e6-9ca8-8576423a1153:staging",
"767401a2-0655-45d6-bd1b-c34edfb46037",
"87205ce6-72ee-48fb-818b-499ab8c92ff1:staging",
"9e487dd6-b0e5-4b61-ac0d-33b9dc46c6f1",
"9e70d7b3-4f04-4988-9fea-d839e29402aa:staging",
"9f2f6525-bd7d-41d5-8cc4-6440aa91b963",
"a24f10e3-444d-4212-8ecd-eebac5737936",
"b2ff6b9d-6d84-401a-a303-4cb74462c491",
"cb8b40da-e0dd-4537-b447-23c6b13b1a48:staging",
"d6fa1761-8f58-4e18-86ef-4541c306f41a",
"e6142e89-9c68-428f-bcc4-ce5084dfa864",
"e77b66db-eca3-4fd4-a987-c2179d178da7",
"f372e083-411b-42a9-a14a-c4fcc7b31b72"
]
}
[rook@rook-ceph-tools-f7f46787d-9jhpg /]$ radosgw-admin sync status
realm 3fc7538d-9a0f-4385-9867-e16200927e9c (realm-a)
zonegroup 003fe63e-b238-400f-aa5e-ecf84b221632 (zonegroup-a)
zone e5f5f214-5f18-4011-a460-aa047000ac7c (zone-a)
metadata sync no sync (zone is master)
data sync source: 2f90434c-2764-48ab-bba3-bcffa042dff1 (zone-b)
syncing
full sync: 0/128 shards
incremental sync: 128/128 shards
data is caught up with source