Last active
March 15, 2024 13:15
-
-
Save frobware/7fdd8b18f580a67ae9cf5ec75b57348b to your computer and use it in GitHub Desktop.
Debug notes for https://issues.redhat.com/browse/OCPBUGS-29690
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# set reload interval to default (5s) | |
# oc patch -n openshift-ingress-operator ingresscontroller/default --type=merge --patch='{"spec":{"tuningOptions":{"reloadInterval":"5s"}}}' | |
# enable router access logs. | |
oc patch -n openshift-ingress-operator ingresscontroller/default --type=merge --patch='{"spec":{"logging":{"access":{"destination":{"type":"Container"}}}}}' | |
# let's concentrate on one router pod only. | |
oc scale -n openshift-ingress-operator ingresscontroller/default --replicas 1 | |
# We're going to replace the router image and change the configuration | |
# so that the router pods runs as user=ROOT. We need to disable CVO | |
# and the ingress-operator to so that our changes don't get | |
# overwritten. | |
oc scale -n openshift-cluster-version deployments/cluster-version-operator --replicas 0 | |
oc scale -n openshift-ingress-operator deployments ingress-operator --replicas 0 | |
# Change cluster configuration to run router pods as ROOT. | |
oc patch clusterversions/version --type=json --patch='[{"op":"add","path":"/spec/overrides","value":[{"kind":"Deployment","group":"apps/v1","name":"ingress-operator","namespace":"openshift-ingress-operator","unmanaged":true}]}]' | |
oc patch clusterrole/openshift-ingress-router --type=strategic --patch='{"rules":[{"apiGroups":[""],"resources":["endpoints","namespaces","services"],"verbs":["list","watch"]},{"apiGroups":["authentication.k8s.io"],"resources":["tokenreviews"],"verbs":["create"]},{"apiGroups":["authorization.k8s.io"],"resources":["subjectaccessreviews"],"verbs":["create"]},{"apiGroups":["route.openshift.io"],"resources":["routes"],"verbs":["list","watch"]},{"apiGroups":["route.openshift.io"],"resources":["routes/status"],"verbs":["update"]},{"apiGroups":["security.openshift.io"],"resourceNames":["privileged"],"resources":["securitycontextconstraints"],"verbs":["use"]},{"apiGroups":["discovery.k8s.io"],"resources":["endpointslices"],"verbs":["list","watch"]}]}' | |
oc patch -n openshift-ingress deployment/router-default --patch='{"spec":{"template":{"spec":{"securityContext":{"runAsUser":0}}}}}' | |
# replace router image. | |
oc -n openshift-ingress patch deployment router-default -p '{"spec":{"template":{"spec":{"$setElementOrder/containers":[{"name":"router"}],"containers":[{"imagePullPolicy":"Always","name":"router"}]}}}}' | |
oc -n openshift-ingress set image deployment/router-default router=quay.io/amcdermo/openshift-router-ocpbugs29690-14march25:latest | |
# let's concentrate on one router pod only. | |
oc scale -n openshift-ingress deployment/router-default --replicas 1 | |
# verify new haproxy version | |
oc -n openshift-ingress rsh -c router deployment/router-default sh -c 'haproxy -v' | |
# oc rsh into a new pod and verify haproxy version; should look like: | |
# sh-4.4# haproxy -v | |
# [60] contrib/interposer/accept.c:289:setup(): ACCEPT INTERPOSER initialised | |
# HA-Proxy version 2.2.24-accept-interposer 2024-03-14T09-24-00-GMT - https://haproxy.org/ | |
# Status: long-term supported branch - will stop receiving fixes around Q2 2025. | |
# Known bugs: http://www.haproxy.org/bugs/bugs-2.2.24.html | |
# Running on: Linux 4.18.0-372.82.1.el8_6.x86_64 #1 SMP Fri Nov 17 17:58:32 EST 2023 x86_64 | |
# Diagnostics / log capture | |
## Getting access to the local stats port; port forward in a shell. | |
oc port-forward -n openshift-ingress deployment/router-default --address localhost 1942:1942 | |
## Run openshift-router to log events related to reloads. | |
oc -n openshift-ingress rsh -c router deploy/router-default sh -c 'rm -rf /tmp/haproxy && mkdir -p /tmp/haproxy/\{router/{certs,cacerts,whitelists},\{conf/.tmp,run,bin,log}} && /bin/openshift-router --v=5 --template=/var/lib/haproxy/conf/haproxy-config.template --reload=/bin/true --update-status=false --stats-port=1937 --metrics-type=haproxy --working-dir=/tmp/haproxy' | tee -a openshift-router-"$(date --iso-8601=seconds)".txt | |
## In another shell gather stats. | |
while :; do curl -s http://localhost:1942 > router-stats-"$(date --iso-8601=seconds)".html; date; sleep 10; done | |
# Cleanup | |
## AND THIS IS HOW TO REVERT EVERYTHING... | |
## AND THIS IS HOW TO REVERT EVERYTHING... | |
# And once we've finished with the interactive debug session we need | |
# to restore everything we have changed. | |
## AND THIS IS HOW TO REVERT EVERYTHING... | |
## AND THIS IS HOW TO REVERT EVERYTHING... | |
oc patch clusterversions/version --type=json --patch='[{"op":"add","path":"/spec/overrides","value":[{"kind":"Deployment","group":"apps/v1","name":"ingress-operator","namespace":"openshift-ingress-operator","unmanaged":false}]}]' | |
oc patch clusterrole/openshift-ingress-router --type=strategic --patch='{"rules":[{"apiGroups":[""],"resources":["endpoints","namespaces","services"],"verbs":["list","watch"]},{"apiGroups":["authentication.k8s.io"],"resources":["tokenreviews"],"verbs":["create"]},{"apiGroups":["authorization.k8s.io"],"resources":["subjectaccessreviews"],"verbs":["create"]},{"apiGroups":["route.openshift.io"],"resources":["routes"],"verbs":["list","watch"]},{"apiGroups":["route.openshift.io"],"resources":["routes/status"],"verbs":["update"]},{"apiGroups":["security.openshift.io"],"resourceNames":["hostnetwork"],"resources":["securitycontextconstraints"],"verbs":["use"]},{"apiGroups":["discovery.k8s.io"],"resources":["endpointslices"],"verbs":["list","watch"]}]}' | |
oc patch -n openshift-ingress deployment/router-default --patch='{"spec":{"template":{"spec":{"securityContext":{"runAsUser":null}}}}}' | |
oc scale --replicas 1 -n openshift-cluster-version deployments/cluster-version-operator | |
oc scale --replicas 1 -n openshift-ingress-operator deployments ingress-operator | |
oc scale --replicas 2 -n openshift-ingress-operator ingresscontroller/default | |
oc patch -n openshift-ingress-operator ingresscontroller/default --type=merge --patch='{"spec":{"logging":null}}' | |
oc patch -n openshift-ingress-operator ingresscontroller/default --type=merge --patch='{"spec":{"tuningOptions":{"reloadInterval":"5s"}}}' |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment