The following readme guides you thru the steps to set up a minimal demo using a local Vault and a 1 node Rook/Ceph cluster hosted in Minikube
$ minikube start (ou minikube start --driver=virtualbox)
When installing rook for the first time, make sure we have a raw device on the minikube host (https://rook.io/docs/rook/v1.3/ceph-quickstart.html)
$ minikube ssh
$ lsblk -f
$ exit
$ git clone --single-branch --branch release-1.3 https://github.com/rook/rook.git
$ cd ./rook/cluster/examples/kubernetes/ceph
$ kubectl create -f common.yaml
$ kubectl create -f operator.yaml
$ kubectl -n rook-ceph get pod
First, update the cluster-test.yaml file with the floowing ceph configuration:
debug rgw = 20/5
rgw crypt s3 kms backend = vault
rgw crypt vault auth = token
rgw crypt vault token file = /ceph/vault.token
rgw crypt vault addr = http://192.168.0.12:8200
rgw crypt vault secret engine = transit
rgw crypt vault prefix = /v1/transit/export/encryption-key
rgw crypt require ssl = false
$ kubectl create -f cluster-test.yaml
$ kubectl -n rook-ceph get pod
$ kubectl logs -n rook-ceph --all-containers rook-ceph-osd-prepare-minikube-dztj7
(you should see that Ceph is using a raw device to create an OSD. If it can't, you will see an error message at the bottom)
$ kubectl create -f object-test.yaml
$ kubectl -n rook-ceph get pod -l app=rook-ceph-rgw
$ kubectl create -f storageclass-bucket-delete.yaml
$ kubectl create -f object-bucket-claim-delete.yaml
$ kubectl -n rook-ceph exec -it rook-ceph-rgw-my-store-a-688c79f8ff-5xtpv /bin/sh
sh-4.4# mkdir /ceph
sh-4.4# echo "s.WK1AULVFXXXXXXXXXX" > /ceph/vault.token
sh-4.4# chmod 600 /ceph/vault.token
sh-4.4# chown ceph /ceph/vault.token
$ vault secrets enable transit
$ vault write -f transit/keys/mybucketkey exportable=true
$ TOKEN=xxx vault read transit/export/encryption-key/mybucketkey/1
$ kubectl -n default get secret ceph-delete-bucket -o yaml | grep AWS_ACCESS_KEY_ID | awk '{print $2}' | base64 --decode
$ kubectl -n default get secret ceph-delete-bucket -o yaml | grep AWS_SECRET_ACCESS_KEY | awk '{print $2}' | base64 --decode
$ kubectl apply -f toolbox.yaml
$ kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}') /bin/sh
# ceph status
# ceph osd status
# radosgw-admin bucket list
[
"rookbucket-6ae7d67e-0e27-42a5-ba00-bb869739b163"
]
# echo "Hello Rook" > /tmp/rookObj
# yum --assumeyes install s3cmd
# s3cmd put /tmp/rookObj --access_key=6GYOI27BO9GWNN87EU72 --secret_key=HslhLOXztccsxEpB6VEdbGYkbDGQ3Vf7wD9Tuyao --no-ssl --host=rook-ceph-rgw-my-store.rook-ceph --server-side-encryption --server-side-encryption-kms-id=mybucketkey/1 --host-bucket= s3://rookbucket-6ae7d67e-0e27-42a5-ba00-bb869739b163
# s3cmd get s3://rookbucket-6ae7d67e-0e27-42a5-ba00-bb869739b163/rookObj /tmp/rookObj-download --access_key=6GYOI27BO9GWNN87EU72 --secret_key=HslhLOXztccsxEpB6VEdbGYkbDGQ3Vf7wD9Tuyao --no-ssl --host=rook-ceph-rgw-my-store.rook-ceph --host-bucket=
# cat /tmp/rookObj-download
(Run the toolbox)
# ceph osd lspools
# rados -p my-store.rgw.buckets.data ls
# rados -p my-store.rgw.buckets.data get 00efb6b5-40ca-4812-9f80-c097de8e512f.4395.1_rookObj /tmp/encrypted
# cat /tmp/encrypted
First, update the cluster-test.yaml file with the floowing ceph configuration:
debug rgw = 20/5
rgw crypt s3 kms backend = vault
rgw crypt vault auth = agent
rgw crypt vault addr = http://localhost:8100
rgw crypt vault secret engine = transit
rgw crypt vault prefix = /v1/transit2/export/encryption-key
rgw crypt require ssl = false
Vault Agent is going to be deplyed as a sidecar container in Pod rgw
Create a configmap (hcl vault config) with the right AuthN method, the right Vault server and Vault agent listener
$ kubectl apply -f vault-agent-config.yaml
Patch the rgw deployment to deploy the sidecar
$ kubectl patch deploy rook-ceph-rgw-my-store-a --patch "$(cat cluster-test-patch.yaml)" -n rook-ceph
See https://learn.hashicorp.com/tutorials/vault/agent-kubernetes
$ kubectl -n rook-ceph logs -f rook-ceph-rgw-my-store-a-688c79f8ff-5xtpv
$ curl --header "X-Vault-Token: s.WK1AUXXXXXXXX" http://192.168.0.12:8200/v1/transit2/export/encryption-key/mybucketkey/1 | jq
$ kubectl create -f object-user.yaml
$ kubectl -n rook-ceph describe secret rook-ceph-object-user-my-store-my-user
$ kubectl -n rook-ceph get secret rook-ceph-object-user-my-store-my-user -o yaml | grep AccessKey | awk '{print $2}' | base64 --decode
$ kubectl -n rook-ceph get secret rook-ceph-object-user-my-store-my-user -o yaml | grep SecretKey | awk '{print $2}' | base64 --decode
$ kubectl port-forward -n rook-ceph svc/rook-ceph-mgr-dashboard 7000:7000
radosgw-admin user modify --uid=my-user --system
ceph dashboard set-rgw-api-user-id my-user
ceph dashboard set-rgw-api-access-key <access-key>
ceph dashboard set-rgw-api-secret-key <secret-key>
$ kubectl delete -f object-user.yaml (if created)
$ kubectl delete -f toolbox.yaml
$ kubectl delete -f object-bucket-claim-delete.yaml
$ kubectl delete -f storageclass-bucket-delete.yaml
$ kubectl delete -f object-test.yaml
$ kubectl delete -f cluster-test.yaml
$ kubectl delete -f operator.yaml
$ kubectl delete -f common.yaml
$ minikube ssh "sudo rm -rf /var/lib/rook"
$ minikube stop
(delete the inital raw device from virtualbox)
References:
- [1] https://rook.io/docs/rook/v1.3/ceph-quickstart.html
- [2] https://rook.io/docs/rook/v1.3/ceph-block.html
- [3] rook/rook#5301
- [4] https://docs.ceph.com/docs/master/radosgw/vault/
- [5] https://blog.zwindler.fr/2019/09/10/du-ceph-dans-mon-kubernetes/
- [6] https://medium.com/@vovaprivalov/setup-and-playing-with-rook-storage-with-minikube-a9424ffcac4b