Skip to content

Instantly share code, notes, and snippets.

@barkbay
Last active August 12, 2022 12:17
Show Gist options
  • Save barkbay/b72b86540093b30bedffafb10f62a323 to your computer and use it in GitHub Desktop.
Save barkbay/b72b86540093b30bedffafb10f62a323 to your computer and use it in GitHub Desktop.
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License 2.0;
// you may not use this file except in compliance with the Elastic License 2.0.
package elasticsearch
import (
"context"
"fmt"
"github.com/pkg/errors"
"testing"
"time"
commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1"
esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1"
"github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/operator"
"github.com/elastic/cloud-on-k8s/v2/pkg/utils/retry"
"github.com/elastic/cloud-on-k8s/v2/pkg/utils/test"
"github.com/stretchr/testify/require"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)
func TestMain(m *testing.M) {
test.RunWithK8s(m)
}
const (
operatorNs = "elastic-system"
elasticsearchResources = 20
)
// go test -v github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch -run '^TestReconcile$' -race -count=1
func TestReconcile(t *testing.T) {
k8sClient, stop := test.StartManager(t,
Add,
operator.Parameters{
OperatorNamespace: operatorNs,
MaxConcurrentReconciles: 20,
},
)
defer stop()
require.NoError(t, test.EnsureNamespace(k8sClient, operatorNs))
require.NoError(t, createElasticsearchResources(k8sClient))
retry.UntilSuccess(
func() error {
t.Helper()
// Try to get the StatefulSes
ssets := &appsv1.StatefulSetList{}
if err := k8sClient.List(context.TODO(), ssets); err != nil {
return err
}
expected := elasticsearchResources
if len(ssets.Items) != expected {
msg := fmt.Sprintf("expected %d StatefulSets, got %d", expected, len(ssets.Items))
return errors.Errorf(msg)
}
return nil
},
time.Second*60, time.Second*1)
}
func createElasticsearchResources(k8sClient client.Client) error {
for i := 1; i <= elasticsearchResources; i++ {
es := &esv1.Elasticsearch{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("es-%d", i),
Namespace: "default",
},
Spec: esv1.ElasticsearchSpec{
Version: "8.0.0",
NodeSets: []esv1.NodeSet{
{
Name: "default",
Config: &commonv1.Config{Data: map[string]interface{}{}},
Count: 10,
},
},
},
}
if err := k8sClient.Create(context.TODO(), es); err != nil {
return err
}
// Built-in controllers are not running, create the service endpoint manually.
endpoints := &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("es-%d-es-internal-http", i),
Namespace: "default",
},
}
if err := k8sClient.Create(context.TODO(), endpoints); err != nil {
return err
}
}
return nil
}
@barkbay
Copy link
Author

barkbay commented Aug 12, 2022

Sample result with -race before elastic/cloud-on-k8s#5947:

go test -v github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch -run '^TestReconcile$' -race -count=1
=== RUN   TestReconcile
==================
WARNING: DATA RACE
Write at 0x00c00070f2f0 by goroutine 749:
  runtime.mapaccess2_faststr()
      /usr/local/go/src/runtime/map_faststr.go:108 +0x43c
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.buildAnnotations()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/podspec.go:196 +0x2c8
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.BuildPodTemplateSpec()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/podspec.go:97 +0xa4c
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.BuildStatefulSet()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/statefulset.go:79 +0x500
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.BuildExpectedResources()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/resources.go:81 +0x49c
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/driver.(*defaultDriver).reconcileNodeSpecs()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/nodes.go:82 +0xdd8
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:349 +0x51a0
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch.(*ReconcileElasticsearch).internalReconcile()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:299 +0x1048
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch.(*ReconcileElasticsearch).Reconcile()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:187 +0x538
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:121 +0xe4
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:320 +0x360
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:273 +0x240
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:234 +0x98

Previous write at 0x00c00070f2f0 by goroutine 758:
  runtime.mapaccess2_faststr()
      /usr/local/go/src/runtime/map_faststr.go:108 +0x43c
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.buildAnnotations()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/podspec.go:196 +0x2c8
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.BuildPodTemplateSpec()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/podspec.go:97 +0xa4c
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.BuildStatefulSet()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/statefulset.go:79 +0x500
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.BuildExpectedResources()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/resources.go:81 +0x49c
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/driver.(*defaultDriver).reconcileNodeSpecs()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/nodes.go:82 +0xdd8
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:349 +0x51a0
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch.(*ReconcileElasticsearch).internalReconcile()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:299 +0x1048
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch.(*ReconcileElasticsearch).Reconcile()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:187 +0x538
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:121 +0xe4
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:320 +0x360
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:273 +0x240
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:234 +0x98

Goroutine 749 (running) created at:
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:230 +0x364
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:241 +0x254
  sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile.func1()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/manager/runnable_group.go:219 +0x158
  sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile.func2()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/manager/runnable_group.go:222 +0x44

Goroutine 758 (running) created at:
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:230 +0x364
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:241 +0x254
  sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile.func1()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/manager/runnable_group.go:219 +0x158
  sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile.func2()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/manager/runnable_group.go:222 +0x44
==================
==================
WARNING: DATA RACE
Write at 0x00c0000003f8 by goroutine 749:
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.buildAnnotations()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/podspec.go:196 +0x2d4
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.BuildPodTemplateSpec()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/podspec.go:97 +0xa4c
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.BuildStatefulSet()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/statefulset.go:79 +0x500
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.BuildExpectedResources()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/resources.go:81 +0x49c
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/driver.(*defaultDriver).reconcileNodeSpecs()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/nodes.go:82 +0xdd8
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:349 +0x51a0
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch.(*ReconcileElasticsearch).internalReconcile()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:299 +0x1048
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch.(*ReconcileElasticsearch).Reconcile()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:187 +0x538
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:121 +0xe4
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:320 +0x360
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:273 +0x240
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:234 +0x98

Previous write at 0x00c0000003f8 by goroutine 758:
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.buildAnnotations()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/podspec.go:196 +0x2d4
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.BuildPodTemplateSpec()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/podspec.go:97 +0xa4c
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.BuildStatefulSet()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/statefulset.go:79 +0x500
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/nodespec.BuildExpectedResources()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec/resources.go:81 +0x49c
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/driver.(*defaultDriver).reconcileNodeSpecs()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/nodes.go:82 +0xdd8
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:349 +0x51a0
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch.(*ReconcileElasticsearch).internalReconcile()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:299 +0x1048
  github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch.(*ReconcileElasticsearch).Reconcile()
      /Users/michael/go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:187 +0x538
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:121 +0xe4
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:320 +0x360
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:273 +0x240
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:234 +0x98

Goroutine 749 (running) created at:
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:230 +0x364
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:241 +0x254
  sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile.func1()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/manager/runnable_group.go:219 +0x158
  sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile.func2()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/manager/runnable_group.go:222 +0x44

Goroutine 758 (running) created at:
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:230 +0x364
  sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/internal/controller/controller.go:241 +0x254
  sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile.func1()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/manager/runnable_group.go:219 +0x158
  sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile.func2()
      /Users/michael/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.12.2/pkg/manager/runnable_group.go:222 +0x44
==================
    testing.go:1319: race detected during execution of test
--- FAIL: TestReconcile (25.09s)
=== CONT
    testing.go:1319: race detected during execution of test
FAIL
E0812 14:12:41.234498   59310 event.go:276] Unable to write event: '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"es-20.170a97d514a2c950", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"793", Generation:0, CreationTimestamp:time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), DeletionTimestamp:<nil>, DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ZZZ_DeprecatedClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Elasticsearch", Namespace:"default", Name:"es-20", UID:"e316a140-8368-41b7-bdc4-3737f0281504", APIVersion:"elasticsearch.k8s.elastic.co/v1", ResourceVersion:"728", FieldPath:""}, Reason:"ReconciliationError", Message:"Reconciliation error: client rate limiter Wait returned an error: context canceled", Source:v1.EventSource{Component:"elasticsearch-controller", Host:""}, FirstTimestamp:time.Date(2022, time.August, 12, 14, 12, 34, 0, time.Local), LastTimestamp:time.Date(2022, time.August, 12, 14, 12, 40, 228596000, time.Local), Count:3, Type:"Warning", EventTime:time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), Series:(*v1.EventSeries)(nil), Action:"", Related:(*v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'Patch "https://127.0.0.1:52416/api/v1/namespaces/default/events/es-20.170a97d514a2c950": dial tcp 127.0.0.1:52416: connect: connection refused'(may retry after sleeping)
FAIL	github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch	30.406s
FAIL

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment