You need to delete contexts, clusters, users seperately.
kubectl config current-context
# Context
kubectl config get-contexts
kubectl config delete-context CONTEXT
# Clusters
kubectl config get-clusters
kubectl config delete-cluster CLUSTER
# Delete all
for i in $(kubectl config get-clusters); do kubectl config delete-cluster $i; done
# Users
kubectl config view -o jsonpath='{range .users[*]}{.name}{"\n"}{end}'
kubectl config unset users.NAME
# Delete all
for i in $(kubectl config view -o jsonpath='{range .users[*]}{.name}{"\n"}{end}'); do kubectl config unset users.$i; done