Skip to content

Commit d82eb78

Browse files
bacherflkhushijain21
bacherfl
authored and
khushijain21
committed
[chore] fix flaky e2e test in k8sclusterreceiver (open-telemetry#37830)
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue. Ex. Adding a feature - Explain what this achieves.--> #### Description This PR addresses the flaky E2E test in the k8sclusterreceiver. Will need to run this multiple times to make sure this solves the issue <!-- Issue number (e.g. open-telemetry#1234) or full URL to issue, if applicable. --> #### Link to tracking issue Fixes open-telemetry#37756 <!--Describe what testing was performed and which tests were added.--> #### Testing Adapted the assertion logic in the existing test --------- Signed-off-by: Florian Bacher <[email protected]>
1 parent 872363f commit d82eb78

File tree

1 file changed

+86
-82
lines changed

1 file changed

+86
-82
lines changed

‎receiver/k8sclusterreceiver/e2e_test.go

+86-82
Original file line numberDiff line numberDiff line change
@@ -76,47 +76,49 @@ func TestE2EClusterScoped(t *testing.T) {
7676
// golden.WriteMetrics(t, expectedFileClusterScoped, metricsConsumer.AllMetrics()[len(metricsConsumer.AllMetrics())-1])
7777
waitForData(t, wantEntries, metricsConsumer)
7878

79-
require.NoError(t, pmetrictest.CompareMetrics(expected, metricsConsumer.AllMetrics()[len(metricsConsumer.AllMetrics())-1],
80-
pmetrictest.IgnoreTimestamp(),
81-
pmetrictest.IgnoreStartTimestamp(),
82-
pmetrictest.IgnoreMetricValues(
83-
"k8s.container.cpu_request",
84-
"k8s.container.memory_limit",
85-
"k8s.container.memory_request",
86-
"k8s.container.restarts",
87-
"k8s.cronjob.active_jobs",
88-
"k8s.deployment.available",
89-
"k8s.deployment.desired",
90-
"k8s.job.active_pods",
91-
"k8s.job.desired_successful_pods",
92-
"k8s.job.failed_pods",
93-
"k8s.job.max_parallel_pods",
94-
"k8s.hpa.current_replicas",
95-
"k8s.job.successful_pods"),
96-
pmetrictest.ChangeResourceAttributeValue("container.id", replaceWithStar),
97-
pmetrictest.ChangeResourceAttributeValue("container.image.name", containerImageShorten),
98-
pmetrictest.ChangeResourceAttributeValue("container.image.tag", replaceWithStar),
99-
pmetrictest.ChangeResourceAttributeValue("k8s.cronjob.uid", replaceWithStar),
100-
pmetrictest.ChangeResourceAttributeValue("k8s.daemonset.uid", replaceWithStar),
101-
pmetrictest.ChangeResourceAttributeValue("k8s.deployment.name", shortenNames),
102-
pmetrictest.ChangeResourceAttributeValue("k8s.deployment.uid", replaceWithStar),
103-
pmetrictest.ChangeResourceAttributeValue("k8s.hpa.uid", replaceWithStar),
104-
pmetrictest.ChangeResourceAttributeValue("k8s.job.name", shortenNames),
105-
pmetrictest.ChangeResourceAttributeValue("k8s.job.uid", replaceWithStar),
106-
pmetrictest.ChangeResourceAttributeValue("k8s.namespace.uid", replaceWithStar),
107-
pmetrictest.ChangeResourceAttributeValue("k8s.node.uid", replaceWithStar),
108-
pmetrictest.ChangeResourceAttributeValue("k8s.pod.name", shortenNames),
109-
pmetrictest.ChangeResourceAttributeValue("k8s.pod.uid", replaceWithStar),
110-
pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.name", shortenNames),
111-
pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.uid", replaceWithStar),
112-
pmetrictest.ChangeResourceAttributeValue("k8s.statefulset.uid", replaceWithStar),
113-
pmetrictest.IgnoreScopeVersion(),
114-
pmetrictest.IgnoreResourceMetricsOrder(),
115-
pmetrictest.IgnoreMetricsOrder(),
116-
pmetrictest.IgnoreScopeMetricsOrder(),
117-
pmetrictest.IgnoreMetricDataPointsOrder(),
118-
),
119-
)
79+
require.EventuallyWithT(t, func(tt *assert.CollectT) {
80+
assert.NoError(tt, pmetrictest.CompareMetrics(expected, metricsConsumer.AllMetrics()[len(metricsConsumer.AllMetrics())-1],
81+
pmetrictest.IgnoreTimestamp(),
82+
pmetrictest.IgnoreStartTimestamp(),
83+
pmetrictest.IgnoreMetricValues(
84+
"k8s.container.cpu_request",
85+
"k8s.container.memory_limit",
86+
"k8s.container.memory_request",
87+
"k8s.container.restarts",
88+
"k8s.cronjob.active_jobs",
89+
"k8s.deployment.available",
90+
"k8s.deployment.desired",
91+
"k8s.job.active_pods",
92+
"k8s.job.desired_successful_pods",
93+
"k8s.job.failed_pods",
94+
"k8s.job.max_parallel_pods",
95+
"k8s.hpa.current_replicas",
96+
"k8s.job.successful_pods"),
97+
pmetrictest.ChangeResourceAttributeValue("container.id", replaceWithStar),
98+
pmetrictest.ChangeResourceAttributeValue("container.image.name", containerImageShorten),
99+
pmetrictest.ChangeResourceAttributeValue("container.image.tag", replaceWithStar),
100+
pmetrictest.ChangeResourceAttributeValue("k8s.cronjob.uid", replaceWithStar),
101+
pmetrictest.ChangeResourceAttributeValue("k8s.daemonset.uid", replaceWithStar),
102+
pmetrictest.ChangeResourceAttributeValue("k8s.deployment.name", shortenNames),
103+
pmetrictest.ChangeResourceAttributeValue("k8s.deployment.uid", replaceWithStar),
104+
pmetrictest.ChangeResourceAttributeValue("k8s.hpa.uid", replaceWithStar),
105+
pmetrictest.ChangeResourceAttributeValue("k8s.job.name", shortenNames),
106+
pmetrictest.ChangeResourceAttributeValue("k8s.job.uid", replaceWithStar),
107+
pmetrictest.ChangeResourceAttributeValue("k8s.namespace.uid", replaceWithStar),
108+
pmetrictest.ChangeResourceAttributeValue("k8s.node.uid", replaceWithStar),
109+
pmetrictest.ChangeResourceAttributeValue("k8s.pod.name", shortenNames),
110+
pmetrictest.ChangeResourceAttributeValue("k8s.pod.uid", replaceWithStar),
111+
pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.name", shortenNames),
112+
pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.uid", replaceWithStar),
113+
pmetrictest.ChangeResourceAttributeValue("k8s.statefulset.uid", replaceWithStar),
114+
pmetrictest.IgnoreScopeVersion(),
115+
pmetrictest.IgnoreResourceMetricsOrder(),
116+
pmetrictest.IgnoreMetricsOrder(),
117+
pmetrictest.IgnoreScopeMetricsOrder(),
118+
pmetrictest.IgnoreMetricDataPointsOrder(),
119+
),
120+
)
121+
}, 3*time.Minute, 1*time.Second)
120122
}
121123

122124
// TestE2ENamespaceScoped tests the k8s cluster receiver with a real k8s cluster.
@@ -160,47 +162,49 @@ func TestE2ENamespaceScoped(t *testing.T) {
160162
// golden.WriteMetrics(t, expectedFileNamespaceScoped, metricsConsumer.AllMetrics()[len(metricsConsumer.AllMetrics())-1])
161163
waitForData(t, wantEntries, metricsConsumer)
162164

163-
require.NoError(t, pmetrictest.CompareMetrics(expected, metricsConsumer.AllMetrics()[len(metricsConsumer.AllMetrics())-1],
164-
pmetrictest.IgnoreTimestamp(),
165-
pmetrictest.IgnoreStartTimestamp(),
166-
pmetrictest.IgnoreMetricValues(
167-
"k8s.container.cpu_request",
168-
"k8s.container.memory_limit",
169-
"k8s.container.memory_request",
170-
"k8s.container.restarts",
171-
"k8s.cronjob.active_jobs",
172-
"k8s.deployment.available",
173-
"k8s.deployment.desired",
174-
"k8s.job.active_pods",
175-
"k8s.job.desired_successful_pods",
176-
"k8s.job.failed_pods",
177-
"k8s.job.max_parallel_pods",
178-
"k8s.hpa.current_replicas",
179-
"k8s.job.successful_pods"),
180-
pmetrictest.ChangeResourceAttributeValue("container.id", replaceWithStar),
181-
pmetrictest.ChangeResourceAttributeValue("container.image.name", containerImageShorten),
182-
pmetrictest.ChangeResourceAttributeValue("container.image.tag", replaceWithStar),
183-
pmetrictest.ChangeResourceAttributeValue("k8s.cronjob.uid", replaceWithStar),
184-
pmetrictest.ChangeResourceAttributeValue("k8s.daemonset.uid", replaceWithStar),
185-
pmetrictest.ChangeResourceAttributeValue("k8s.deployment.name", shortenNames),
186-
pmetrictest.ChangeResourceAttributeValue("k8s.deployment.uid", replaceWithStar),
187-
pmetrictest.ChangeResourceAttributeValue("k8s.hpa.uid", replaceWithStar),
188-
pmetrictest.ChangeResourceAttributeValue("k8s.job.name", shortenNames),
189-
pmetrictest.ChangeResourceAttributeValue("k8s.job.uid", replaceWithStar),
190-
pmetrictest.ChangeResourceAttributeValue("k8s.namespace.uid", replaceWithStar),
191-
pmetrictest.ChangeResourceAttributeValue("k8s.node.uid", replaceWithStar),
192-
pmetrictest.ChangeResourceAttributeValue("k8s.pod.name", shortenNames),
193-
pmetrictest.ChangeResourceAttributeValue("k8s.pod.uid", replaceWithStar),
194-
pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.name", shortenNames),
195-
pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.uid", replaceWithStar),
196-
pmetrictest.ChangeResourceAttributeValue("k8s.statefulset.uid", replaceWithStar),
197-
pmetrictest.IgnoreScopeVersion(),
198-
pmetrictest.IgnoreResourceMetricsOrder(),
199-
pmetrictest.IgnoreMetricsOrder(),
200-
pmetrictest.IgnoreScopeMetricsOrder(),
201-
pmetrictest.IgnoreMetricDataPointsOrder(),
202-
),
203-
)
165+
require.EventuallyWithT(t, func(tt *assert.CollectT) {
166+
assert.NoError(tt, pmetrictest.CompareMetrics(expected, metricsConsumer.AllMetrics()[len(metricsConsumer.AllMetrics())-1],
167+
pmetrictest.IgnoreTimestamp(),
168+
pmetrictest.IgnoreStartTimestamp(),
169+
pmetrictest.IgnoreMetricValues(
170+
"k8s.container.cpu_request",
171+
"k8s.container.memory_limit",
172+
"k8s.container.memory_request",
173+
"k8s.container.restarts",
174+
"k8s.cronjob.active_jobs",
175+
"k8s.deployment.available",
176+
"k8s.deployment.desired",
177+
"k8s.job.active_pods",
178+
"k8s.job.desired_successful_pods",
179+
"k8s.job.failed_pods",
180+
"k8s.job.max_parallel_pods",
181+
"k8s.hpa.current_replicas",
182+
"k8s.job.successful_pods"),
183+
pmetrictest.ChangeResourceAttributeValue("container.id", replaceWithStar),
184+
pmetrictest.ChangeResourceAttributeValue("container.image.name", containerImageShorten),
185+
pmetrictest.ChangeResourceAttributeValue("container.image.tag", replaceWithStar),
186+
pmetrictest.ChangeResourceAttributeValue("k8s.cronjob.uid", replaceWithStar),
187+
pmetrictest.ChangeResourceAttributeValue("k8s.daemonset.uid", replaceWithStar),
188+
pmetrictest.ChangeResourceAttributeValue("k8s.deployment.name", shortenNames),
189+
pmetrictest.ChangeResourceAttributeValue("k8s.deployment.uid", replaceWithStar),
190+
pmetrictest.ChangeResourceAttributeValue("k8s.hpa.uid", replaceWithStar),
191+
pmetrictest.ChangeResourceAttributeValue("k8s.job.name", shortenNames),
192+
pmetrictest.ChangeResourceAttributeValue("k8s.job.uid", replaceWithStar),
193+
pmetrictest.ChangeResourceAttributeValue("k8s.namespace.uid", replaceWithStar),
194+
pmetrictest.ChangeResourceAttributeValue("k8s.node.uid", replaceWithStar),
195+
pmetrictest.ChangeResourceAttributeValue("k8s.pod.name", shortenNames),
196+
pmetrictest.ChangeResourceAttributeValue("k8s.pod.uid", replaceWithStar),
197+
pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.name", shortenNames),
198+
pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.uid", replaceWithStar),
199+
pmetrictest.ChangeResourceAttributeValue("k8s.statefulset.uid", replaceWithStar),
200+
pmetrictest.IgnoreScopeVersion(),
201+
pmetrictest.IgnoreResourceMetricsOrder(),
202+
pmetrictest.IgnoreMetricsOrder(),
203+
pmetrictest.IgnoreScopeMetricsOrder(),
204+
pmetrictest.IgnoreMetricDataPointsOrder(),
205+
),
206+
)
207+
}, 3*time.Minute, 1*time.Second)
204208
}
205209

206210
func shortenNames(value string) string {

0 commit comments

Comments
 (0)