Skip to content

Commit a04b304

Browse files
ChrsMarkAkhigbeEromo
authored andcommitted
[receiver/receiver_creator] Add support for enabling logs' collecting from K8s hints (open-telemetry#36581)
#### Description This PR adds the logs part for open-telemetry#34427 based on the design decided at open-telemetry#34427 (comment). See the README docs for the description of this feature: https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/35617/files#diff-4127365c4062a7510fb7fede0fa239e9232549732898303d94c12fef0433d39d #### Link to tracking issue Fixes open-telemetry#34427 #### Testing Added unit-tests #### Documentation Added README section #### How to test this manually 1. Deploy the Collector helm chart: ```yaml mode: daemonset image: repository: otelcontribcol-dev tag: "latest" pullPolicy: IfNotPresent command: name: otelcontribcol clusterRole: create: true rules: - apiGroups: - '' resources: - 'pods' - 'nodes' verbs: - 'get' - 'list' - 'watch' - apiGroups: [ "" ] resources: [ "nodes/proxy"] verbs: [ "get" ] - apiGroups: - "" resources: - nodes/stats verbs: - get - nonResourceURLs: - "/metrics" verbs: - get extraVolumeMounts: - name: varlogpods mountPath: /var/log/pods readOnly: true extraVolumes: - name: varlogpods hostPath: path: /var/log/pods config: extensions: k8s_observer: auth_type: serviceAccount node: ${env:K8S_NODE_NAME} observe_nodes: true exporters: debug: verbosity: detailed receivers: receiver_creator/metrics: watch_observers: [ k8s_observer ] discovery: enabled: true ignore_receivers: - nginx2 receivers: receiver_creator/logs: watch_observers: [ k8s_observer ] discovery: enabled: true default_logs_discovery: false receivers: service: extensions: [health_check, k8s_observer] telemetry: logs: level: INFO pipelines: metrics: receivers: [ receiver_creator/metrics ] processors: [ batch ] exporters: [ debug ] logs/discovery: receivers: [ receiver_creator/logs ] #processors: [ batch ] exporters: [ debug ] ``` 2. Then deploy a target Pod with 2 containers: ```yaml apiVersion: apps/v1 kind: Deployment metadata: name: redis-deployment labels: app: redis spec: replicas: 1 selector: matchLabels: app: redis template: metadata: labels: app: redis annotations: io.opentelemetry.discovery.metrics.6379/enabled: "true" io.opentelemetry.discovery.metrics.6379/scraper: redis io.opentelemetry.discovery.metrics.6379/signals: metrics io.opentelemetry.discovery.metrics.6379/config: | collection_interval: "20s" timeout: "10s" io.opentelemetry.discovery.logs.busybox/enabled: "true" io.opentelemetry.discovery.logs.busybox/config: | operators: - id: some type: add field: attributes.tag value: hints spec: containers: - image: redis imagePullPolicy: IfNotPresent name: redis ports: - name: redis containerPort: 6379 protocol: TCP - name: busybox image: busybox args: - /bin/sh - -c - while true; do echo "otel logs at $(date +%H:%M:%S)" && sleep 15s; done ``` 3. Esnure that logs are collected from both containers and that Redis metrics are collected from the Redis container: ```console 2024-11-28T11:04:14.921Z info [email protected]/observerhandler.go:201 starting receiver {"kind": "receiver", "name": "receiver_creator/metrics", "data_type": "metrics", "name": "redis/91ec7d5c-c6fb-4977-9dbb-c24a85101326_6379", "endpoint": "10.244.0.6:6379", "endpoint_id": "k8s_observer/91ec7d5c-c6fb-4977-9dbb-c24a85101326/redis(6379)", "config": {"collection_interval":"20s","endpoint":"10.244.0.6:6379","timeout":"10s"}} 2024-11-28T11:04:14.921Z info [email protected]/observerhandler.go:201 starting receiver {"kind": "receiver", "name": "receiver_creator/logs", "data_type": "logs", "name": "filelog/91ec7d5c-c6fb-4977-9dbb-c24a85101326_busybox", "endpoint": "10.244.0.6", "endpoint_id": "k8s_observer/91ec7d5c-c6fb-4977-9dbb-c24a85101326/busybox", "config": {"include":["/var/log/pods/default_redis-deployment-7777bf7db4-5rm6d_91ec7d5c-c6fb-4977-9dbb-c24a85101326/busybox/*.log"],"include_file_name":false,"include_file_path":true,"operators":[{"id":"container-parser","type":"container"},{"field":"attributes.tag","id":"some","type":"add","value":"hints"}]}} 2024-11-28T11:04:14.922Z info adapter/receiver.go:41 Starting stanza receiver {"kind": "receiver", "name": "receiver_creator/logs", "data_type": "logs", "name": "filelog/91ec7d5c-c6fb-4977-9dbb-c24a85101326_busybox/receiver_creator/logs{endpoint=\"10.244.0.6\"}/k8s_observer/91ec7d5c-c6fb-4977-9dbb-c24a85101326/busybox"} 2024-11-28T11:04:15.122Z info fileconsumer/file.go:265 Started watching file {"kind": "receiver", "name": "receiver_creator/logs", "data_type": "logs", "name": "filelog/91ec7d5c-c6fb-4977-9dbb-c24a85101326_busybox/receiver_creator/logs{endpoint=\"10.244.0.6\"}/k8s_observer/91ec7d5c-c6fb-4977-9dbb-c24a85101326/busybox", "component": "fileconsumer", "path": "/var/log/pods/default_redis-deployment-7777bf7db4-5rm6d_91ec7d5c-c6fb-4977-9dbb-c24a85101326/busybox/0.log"} 2024-11-28T11:04:15.979Z info Metrics {"kind": "exporter", "data_type": "metrics", "name": "debug/2", "resource metrics": 1, "metrics": 26, "data points": 31} ``` ### Follow-ups 1. File an issue for enhancing default behaviors: open-telemetry#36581 (comment) --------- Signed-off-by: ChrsMark <[email protected]>
1 parent 5840445 commit a04b304

File tree

8 files changed

+714
-15
lines changed

8 files changed

+714
-15
lines changed

.chloggen/f_hints_logs.yaml

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Use this changelog template to create an entry for release notes.
2+
3+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
4+
change_type: enhancement
5+
6+
# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
7+
component: receivercreator
8+
9+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
10+
note: Add support for starting logs' collection based on provided k8s annotations' hints
11+
12+
# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
13+
issues: [34427]
14+
15+
# (Optional) One or more lines of additional information to render under the primary note.
16+
# These lines will be padded with 2 spaces and then inserted directly into the document.
17+
# Use pipe (|) for multiline entries.
18+
subtext:
19+
20+
# If your change doesn't affect end users or the exported elements of any package,
21+
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
22+
# Optional: The change log or logs in which this entry should be included.
23+
# e.g. '[user]' or '[user, api]'
24+
# Include 'user' if the change is relevant to end users.
25+
# Include 'api' if there is a change to a library API.
26+
# Default: '[user]'
27+
change_logs: [user]

receiver/receivercreator/README.md

+96-3
Original file line numberDiff line numberDiff line change
@@ -458,7 +458,8 @@ receiver_creator/metrics:
458458
# ignore_receivers: []
459459
```
460460

461-
Find bellow the supported annotations that user can define to automatically enable receivers to start collecting metrics signals from the target Pods/containers.
461+
Find bellow the supported annotations that user can define to automatically enable receivers to start
462+
collecting metrics and logs signals from the target Pods/containers.
462463

463464
### Supported metrics annotations
464465

@@ -511,11 +512,76 @@ The current implementation relies on the implementation of `k8sobserver` extensi
511512
the [pod_endpoint](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.111.0/extension/observer/k8sobserver/pod_endpoint.go).
512513
The hints are evaluated per container by extracting the annotations from each [`Port` endpoint](#Port) that is emitted.
513514

515+
### Supported logs annotations
516+
517+
This feature enables `filelog` receiver in order to collect logs from the discovered Pods.
518+
519+
#### Enable/disable discovery
520+
521+
`io.opentelemetry.discovery.logs/enabled` (Required. Example: `"true"`)
522+
523+
By default `"false"`.
524+
525+
#### Define configuration
526+
527+
The default configuration for the `filelog` receiver is the following:
528+
529+
```yaml
530+
include:
531+
- /var/log/pods/`pod.namespace`_`pod.name`_`pod.uid`/`container_name`/*.log
532+
include_file_name: false
533+
include_file_path: true
534+
operators:
535+
- id: container-parser
536+
type: container
537+
```
538+
This default can be extended or overridden using the respective annotation:
539+
`io.opentelemetry.discovery.logs/config`
540+
541+
**Example:**
542+
543+
```yaml
544+
io.opentelemetry.discovery.logs/config: |
545+
include_file_name: true
546+
max_log_size: "2MiB"
547+
operators:
548+
- type: container
549+
id: container-parser
550+
- type: regex_parser
551+
regex: "^(?P<time>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}) (?P<sev>[A-Z]*) (?P<msg>.*)$"
552+
```
553+
554+
`include` cannot be overridden and is fixed to discovered container's log file path.
555+
556+
#### Support multiple target containers
557+
558+
Users can target the annotation to a specific container by suffixing it with the name of that container:
559+
`io.opentelemetry.discovery.logs.<container_name>/endpoint`.
560+
For example:
561+
```yaml
562+
io.opentelemetry.discovery.logs.busybox/config: |
563+
max_log_size: "3MiB"
564+
operators:
565+
- type: container
566+
id: container-parser
567+
- id: some
568+
type: add
569+
field: attributes.tag
570+
value: hints
571+
```
572+
where `busybox` is the name of the target container.
573+
574+
If a Pod is annotated with both container level hints and pod level hints the container level hints have priority and
575+
the Pod level hints are used as a fallback (see detailed example bellow).
576+
577+
The current implementation relies on the implementation of `k8sobserver` extension and specifically
578+
the [pod_endpoint](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.111.0/extension/observer/k8sobserver/pod_endpoint.go).
579+
The hints are evaluated per container by extracting the annotations from each [`Pod Container` endpoint](#Pod Container) that is emitted.
514580

515581

516582
### Examples
517583

518-
#### Metrics example
584+
#### Metrics and Logs example
519585

520586
Collector's configuration:
521587
```yaml
@@ -525,12 +591,22 @@ receivers:
525591
discovery:
526592
enabled: true
527593
receivers:
594+
595+
receiver_creator/logs:
596+
watch_observers: [ k8s_observer ]
597+
discovery:
598+
enabled: true
599+
receivers:
528600
529601
service:
530602
extensions: [ k8s_observer]
531603
pipelines:
532604
metrics:
533-
receivers: [ receiver_creator ]
605+
receivers: [ receiver_creator/metrics ]
606+
processors: []
607+
exporters: [ debug ]
608+
logs:
609+
receivers: [ receiver_creator/logs ]
534610
processors: []
535611
exporters: [ debug ]
536612
```
@@ -600,6 +676,23 @@ spec:
600676
endpoint: "http://`endpoint`/nginx_status"
601677
collection_interval: "30s"
602678
timeout: "20s"
679+
680+
# redis pod container logs hints
681+
io.opentelemetry.discovery.logs.redis/enabled: "true"
682+
io.opentelemetry.discovery.logs.redis/config: |
683+
max_log_size: "4MiB"
684+
operators:
685+
- type: container
686+
id: container-parser
687+
- id: some
688+
type: add
689+
field: attributes.tag
690+
value: logs_hints
691+
692+
# nginx pod container logs hints
693+
io.opentelemetry.discovery.logs.webserver/enabled: "true"
694+
io.opentelemetry.discovery.logs.webserver/config: |
695+
max_log_size: "3MiB"
603696
spec:
604697
volumes:
605698
- name: nginx-conf

receiver/receivercreator/config_test.go

+37
Original file line numberDiff line numberDiff line change
@@ -299,3 +299,40 @@ func (*nopWithoutEndpointFactory) CreateTraces(
299299
cfg: cfg,
300300
}, nil
301301
}
302+
303+
type nopWithFilelogConfig struct {
304+
Include []string `mapstructure:"include"`
305+
IncludeFileName bool `mapstructure:"include_file_name"`
306+
IncludeFilePath bool `mapstructure:"include_file_path"`
307+
Operators []any `mapstructure:"operators"`
308+
}
309+
310+
type nopWithFilelogFactory struct {
311+
rcvr.Factory
312+
}
313+
314+
type nopWithFilelogReceiver struct {
315+
mockComponent
316+
consumer.Logs
317+
consumer.Metrics
318+
consumer.Traces
319+
rcvr.Settings
320+
cfg component.Config
321+
}
322+
323+
func (*nopWithFilelogFactory) CreateDefaultConfig() component.Config {
324+
return &nopWithFilelogConfig{}
325+
}
326+
327+
func (*nopWithFilelogFactory) CreateLogs(
328+
_ context.Context,
329+
rcs rcvr.Settings,
330+
cfg component.Config,
331+
nextConsumer consumer.Logs,
332+
) (rcvr.Logs, error) {
333+
return &nopWithEndpointReceiver{
334+
Logs: nextConsumer,
335+
Settings: rcs,
336+
cfg: cfg,
337+
}, nil
338+
}

receiver/receivercreator/discovery.go

+94-4
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,15 @@ const (
2121

2222
// hint suffix for metrics
2323
otelMetricsHints = otelHints + ".metrics"
24+
otelLogsHints = otelHints + ".logs"
2425

2526
// hints definitions
2627
discoveryEnabledHint = "enabled"
2728
scraperHint = "scraper"
2829
configHint = "config"
30+
31+
logsReceiver = "filelog"
32+
defaultLogPathPattern = "/var/log/pods/%s_%s_%s/%s/*.log"
2933
)
3034

3135
// k8sHintsBuilder creates configurations from hints provided as Pod's annotations.
@@ -57,7 +61,7 @@ func (builder *k8sHintsBuilder) createReceiverTemplateFromHints(env observer.End
5761
return nil, fmt.Errorf("could not get endpoint type: %v", zap.Any("env", env))
5862
}
5963

60-
if endpointType != string(observer.PortType) {
64+
if endpointType != string(observer.PortType) && endpointType != string(observer.PodContainerType) {
6165
return nil, nil
6266
}
6367

@@ -72,7 +76,14 @@ func (builder *k8sHintsBuilder) createReceiverTemplateFromHints(env observer.End
7276
return nil, nil
7377
}
7478

75-
return builder.createScraper(pod.Annotations, env)
79+
switch endpointType {
80+
case string(observer.PortType):
81+
return builder.createScraper(pod.Annotations, env)
82+
case string(observer.PodContainerType):
83+
return builder.createLogsReceiver(pod.Annotations, env)
84+
default:
85+
return nil, nil
86+
}
7687
}
7788

7889
func (builder *k8sHintsBuilder) createScraper(
@@ -91,7 +102,7 @@ func (builder *k8sHintsBuilder) createScraper(
91102
port = p.Port
92103
pod := p.Pod
93104

94-
if !discoveryMetricsEnabled(annotations, otelMetricsHints, fmt.Sprint(port)) {
105+
if !discoveryEnabled(annotations, otelMetricsHints, fmt.Sprint(port)) {
95106
return nil, nil
96107
}
97108

@@ -118,6 +129,48 @@ func (builder *k8sHintsBuilder) createScraper(
118129
return &recTemplate, err
119130
}
120131

132+
func (builder *k8sHintsBuilder) createLogsReceiver(
133+
annotations map[string]string,
134+
env observer.EndpointEnv,
135+
) (*receiverTemplate, error) {
136+
if _, ok := builder.ignoreReceivers[logsReceiver]; ok {
137+
// receiver is ignored
138+
return nil, nil
139+
}
140+
141+
var containerName string
142+
var c observer.PodContainer
143+
err := mapstructure.Decode(env, &c)
144+
if err != nil {
145+
return nil, fmt.Errorf("could not extract pod's container: %v", zap.Any("env", env))
146+
}
147+
if c.Name == "" {
148+
return nil, fmt.Errorf("could not extract container name: %v", zap.Any("container", c))
149+
}
150+
containerName = c.Name
151+
pod := c.Pod
152+
153+
if !discoveryEnabled(annotations, otelLogsHints, containerName) {
154+
return nil, nil
155+
}
156+
157+
subreceiverKey := logsReceiver
158+
builder.logger.Debug("handling added hinted receiver", zap.Any("subreceiverKey", subreceiverKey))
159+
160+
userConfMap := createLogsConfig(
161+
annotations,
162+
containerName,
163+
pod.UID,
164+
pod.Name,
165+
pod.Namespace,
166+
builder.logger)
167+
168+
recTemplate, err := newReceiverTemplate(fmt.Sprintf("%v/%v_%v", subreceiverKey, pod.UID, containerName), userConfMap)
169+
recTemplate.signals = receiverSignals{metrics: false, logs: true, traces: false}
170+
171+
return &recTemplate, err
172+
}
173+
121174
func getScraperConfFromAnnotations(
122175
annotations map[string]string,
123176
defaultEndpoint, scopeSuffix string,
@@ -149,6 +202,43 @@ func getScraperConfFromAnnotations(
149202
return conf, nil
150203
}
151204

205+
func createLogsConfig(
206+
annotations map[string]string,
207+
containerName, podUID, podName, namespace string,
208+
logger *zap.Logger,
209+
) userConfigMap {
210+
scopeSuffix := containerName
211+
logPath := fmt.Sprintf(defaultLogPathPattern, namespace, podName, podUID, containerName)
212+
cont := []any{map[string]any{"id": "container-parser", "type": "container"}}
213+
defaultConfMap := userConfigMap{
214+
"include": []string{logPath},
215+
"include_file_path": true,
216+
"include_file_name": false,
217+
"operators": cont,
218+
}
219+
220+
configStr, found := getHintAnnotation(annotations, otelLogsHints, configHint, scopeSuffix)
221+
if !found || configStr == "" {
222+
return defaultConfMap
223+
}
224+
225+
userConf := make(map[string]any)
226+
if err := yaml.Unmarshal([]byte(configStr), &userConf); err != nil {
227+
logger.Debug("could not unmarshal configuration from hint", zap.Error(err))
228+
}
229+
230+
for k, v := range userConf {
231+
if k == "include" {
232+
// path cannot be other than the one of the target container
233+
logger.Warn("include setting cannot be set through annotation's hints")
234+
continue
235+
}
236+
defaultConfMap[k] = v
237+
}
238+
239+
return defaultConfMap
240+
}
241+
152242
func getHintAnnotation(annotations map[string]string, hintBase string, hintKey string, suffix string) (string, bool) {
153243
// try to scope the hint more on container level by suffixing
154244
// with .<port> in case of Port event or # TODO: .<container_name> in case of Pod Container event
@@ -162,7 +252,7 @@ func getHintAnnotation(annotations map[string]string, hintBase string, hintKey s
162252
return podLevelHint, ok
163253
}
164254

165-
func discoveryMetricsEnabled(annotations map[string]string, hintBase string, scopeSuffix string) bool {
255+
func discoveryEnabled(annotations map[string]string, hintBase string, scopeSuffix string) bool {
166256
enabledHint, found := getHintAnnotation(annotations, hintBase, discoveryEnabledHint, scopeSuffix)
167257
if !found {
168258
return false

0 commit comments

Comments
 (0)