Skip to content

Commit 0e505d5

Browse files
author
Paulo Janotti
authored
Refactor config: pipelines now under service (#376)
* Refactor config: pipelines now under service This change moves 'pipelines' sections from the top level to under the 'service' section. * Fix typo in old comment
1 parent 402b80c commit 0e505d5

File tree

84 files changed

+555
-486
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

84 files changed

+555
-486
lines changed

README.md

+8-7
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,8 @@ exporters:
139139
...
140140
processors:
141141
...
142-
pipelines:
142+
service:
143+
pipelines:
143144
...
144145
```
145146

@@ -218,12 +219,12 @@ a receiver/exporter is reference by all the pipelines.
218219
The following is an example pipeline configuration. For more information, refer
219220
to [pipeline documentation](docs/pipelines.md)
220221
```yaml
221-
pipelines:
222-
traces:
223-
receivers: [examplereceiver]
224-
processors: [exampleprocessor]
225-
exporters: [exampleexporter]
226-
222+
service:
223+
pipelines:
224+
traces:
225+
receivers: [examplereceiver]
226+
processors: [exampleprocessor]
227+
exporters: [exampleexporter]
227228
```
228229
229230
### <a name="config-diagnostics"></a>Diagnostics

config/config.go

+23-15
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,6 @@ func Load(
137137
Receivers map[string]interface{} `mapstructure:"receivers"`
138138
Processors map[string]interface{} `mapstructure:"processors"`
139139
Exporters map[string]interface{} `mapstructure:"exporters"`
140-
Pipelines map[string]interface{} `mapstructure:"pipelines"`
141140
}
142141

143142
if err := v.UnmarshalExact(&topLevelSections); err != nil {
@@ -147,21 +146,15 @@ func Load(
147146
}
148147
}
149148

150-
// Start with extensions and service.
149+
// Start with the service extensions.
151150

152151
extensions, err := loadExtensions(v, factories.Extensions)
153152
if err != nil {
154153
return nil, err
155154
}
156155
config.Extensions = extensions
157156

158-
service, err := loadService(v)
159-
if err != nil {
160-
return nil, err
161-
}
162-
config.Service = service
163-
164-
// Load data components (receivers, exporters, processores, and pipelines).
157+
// Load data components (receivers, exporters, and processors).
165158

166159
receivers, err := loadReceivers(v, factories.Receivers)
167160
if err != nil {
@@ -181,11 +174,12 @@ func Load(
181174
}
182175
config.Processors = processors
183176

184-
pipelines, err := loadPipelines(v)
177+
// Load the service and its data pipelines.
178+
service, err := loadService(v)
185179
if err != nil {
186180
return nil, err
187181
}
188-
config.Pipelines = pipelines
182+
config.Service = service
189183

190184
// Config is loaded. Now validate it.
191185

@@ -296,13 +290,27 @@ func loadExtensions(v *viper.Viper, factories map[string]extension.Factory) (con
296290

297291
func loadService(v *viper.Viper) (configmodels.Service, error) {
298292
var service configmodels.Service
299-
if err := v.UnmarshalKey(serviceKeyName, &service, errorOnUnused); err != nil {
293+
serviceSub := getConfigSection(v, serviceKeyName)
294+
295+
// Process the pipelines first so in case of error on them it can be properly
296+
// reported.
297+
pipelines, err := loadPipelines(serviceSub)
298+
if err != nil {
299+
return service, err
300+
}
301+
302+
// Do an exact match to find any unused section on config.
303+
if err := serviceSub.UnmarshalExact(&service); err != nil {
300304
return service, &configError{
301305
code: errUnmarshalErrorOnService,
302306
msg: fmt.Sprintf("error reading settings for %q: %v", serviceKeyName, err),
303307
}
304308
}
305309

310+
// Unmarshal cannot properly build Pipelines field, set it to the value
311+
// previously loaded.
312+
service.Pipelines = pipelines
313+
306314
return service, nil
307315
}
308316

@@ -640,12 +648,12 @@ func validateServiceExtensions(
640648

641649
func validatePipelines(cfg *configmodels.Config, logger *zap.Logger) error {
642650
// Must have at least one pipeline.
643-
if len(cfg.Pipelines) < 1 {
651+
if len(cfg.Service.Pipelines) < 1 {
644652
return &configError{code: errMissingPipelines, msg: "must have at least one pipeline"}
645653
}
646654

647655
// Validate pipelines.
648-
for _, pipeline := range cfg.Pipelines {
656+
for _, pipeline := range cfg.Service.Pipelines {
649657
if err := validatePipeline(cfg, pipeline, logger); err != nil {
650658
return err
651659
}
@@ -849,7 +857,7 @@ func validateProcessors(cfg *configmodels.Config) {
849857
// getConfigSection returns a sub-config from the viper config that has the corresponding given key.
850858
// It also expands all the string values.
851859
func getConfigSection(v *viper.Viper, key string) *viper.Viper {
852-
// Unmarsh only the subconfig for this processor.
860+
// Unmarshal only the subconfig for this processor.
853861
sv := v.Sub(key)
854862
if sv == nil {
855863
// When the config for this key is empty Sub returns nil. In order to avoid nil checks

config/config_test.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ func TestDecodeConfig(t *testing.T) {
111111
"Did not load processor config correctly")
112112

113113
// Verify Pipelines
114-
assert.Equal(t, 1, len(config.Pipelines), "Incorrect pipelines count")
114+
assert.Equal(t, 1, len(config.Service.Pipelines), "Incorrect pipelines count")
115115

116116
assert.Equal(t,
117117
&configmodels.Pipeline{
@@ -121,7 +121,7 @@ func TestDecodeConfig(t *testing.T) {
121121
Processors: []string{"exampleprocessor"},
122122
Exporters: []string{"exampleexporter"},
123123
},
124-
config.Pipelines["traces"],
124+
config.Service.Pipelines["traces"],
125125
"Did not load pipeline config correctly")
126126
}
127127

@@ -275,7 +275,7 @@ func TestSimpleConfig(t *testing.T) {
275275
"TEST[%s] Did not load processor config correctly", test.name)
276276

277277
// Verify Pipelines
278-
assert.Equalf(t, 1, len(config.Pipelines), "TEST[%s]", test.name)
278+
assert.Equalf(t, 1, len(config.Service.Pipelines), "TEST[%s]", test.name)
279279

280280
assert.Equalf(t,
281281
&configmodels.Pipeline{
@@ -285,7 +285,7 @@ func TestSimpleConfig(t *testing.T) {
285285
Processors: []string{"exampleprocessor"},
286286
Exporters: []string{"exampleexporter"},
287287
},
288-
config.Pipelines["traces"],
288+
config.Service.Pipelines["traces"],
289289
"TEST[%s] Did not load pipeline config correctly", test.name)
290290
}
291291
}

config/configmodels/configmodels.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ type Config struct {
3636
Receivers Receivers
3737
Exporters Exporters
3838
Processors Processors
39-
Pipelines Pipelines
4039
Extensions Extensions
4140
Service Service
4241
}
@@ -144,6 +143,9 @@ type Extensions map[string]Extension
144143
type Service struct {
145144
// Extensions is the ordered list of extensions configured for the service.
146145
Extensions []string `mapstructure:"extensions"`
146+
147+
// Pipelines is the set of data pipelines configured for the service.
148+
Pipelines Pipelines `mapstructure:"pipelines"`
147149
}
148150

149151
// Below are common setting structs for Receivers, Exporters and Processors.

config/testdata/duplicate-exporter.yaml

+6-5
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,9 @@ exporters:
55
exampleexporter/ exp :
66
processors:
77
exampleprocessor:
8-
pipelines:
9-
traces:
10-
receivers: [examplereceiver]
11-
exporters: [exampleexporter]
12-
processors: [exampleprocessor]
8+
service:
9+
pipelines:
10+
traces:
11+
receivers: [examplereceiver]
12+
exporters: [exampleexporter]
13+
processors: [exampleprocessor]

config/testdata/duplicate-pipeline.yaml

+10-9
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,13 @@ exporters:
44
exampleexporter:
55
processors:
66
exampleprocessor:
7-
pipelines:
8-
traces/default:
9-
receivers: [examplereceiver]
10-
exporters: [exampleexporter]
11-
processors: [exampleprocessor]
12-
traces/ default:
13-
receivers: [examplereceiver]
14-
exporters: [exampleexporter]
15-
processors: [exampleprocessor]
7+
service:
8+
pipelines:
9+
traces/default:
10+
receivers: [examplereceiver]
11+
exporters: [exampleexporter]
12+
processors: [exampleprocessor]
13+
traces/ default:
14+
receivers: [examplereceiver]
15+
exporters: [exampleexporter]
16+
processors: [exampleprocessor]

config/testdata/duplicate-processor.yaml

+6-5
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,9 @@ exporters:
55
processors:
66
exampleprocessor/ abc:
77
exampleprocessor/abc:
8-
pipelines:
9-
traces:
10-
receivers: [examplereceiver]
11-
exporters: [exampleexporter]
12-
processors: [exampleprocessor]
8+
service:
9+
pipelines:
10+
traces:
11+
receivers: [examplereceiver]
12+
exporters: [exampleexporter]
13+
processors: [exampleprocessor]

config/testdata/duplicate-receiver.yaml

+6-5
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,9 @@ exporters:
55
exampleexporter:
66
processors:
77
exampleprocessor:
8-
pipelines:
9-
traces:
10-
receivers: [examplereceiver]
11-
exporters: [exampleexporter]
12-
processors: [exampleprocessor]
8+
service:
9+
pipelines:
10+
traces:
11+
receivers: [examplereceiver]
12+
exporters: [exampleexporter]
13+
processors: [exampleprocessor]

config/testdata/invalid-bool-value.yaml

+6-5
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,9 @@ exporters:
55
processors:
66
exampleprocessor:
77
disabled: [2,3]
8-
pipelines:
9-
traces:
10-
receivers: [examplereceiver]
11-
exporters: [exampleexporter]
12-
processors: [exampleprocessor]
8+
service:
9+
pipelines:
10+
traces:
11+
receivers: [examplereceiver]
12+
exporters: [exampleexporter]
13+
processors: [exampleprocessor]

config/testdata/invalid-disabled-bool-value.yaml

+6-5
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,9 @@ exporters:
55
disabled: "string for bool"
66
processors:
77
exampleprocessor:
8-
pipelines:
9-
traces:
10-
receivers: [examplereceiver]
11-
exporters: [exampleexporter]
12-
processors: [exampleprocessor]
8+
service:
9+
pipelines:
10+
traces:
11+
receivers: [examplereceiver]
12+
exporters: [exampleexporter]
13+
processors: [exampleprocessor]

config/testdata/invalid-disabled-bool-value2.yaml

+6-5
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,9 @@ exporters:
55
exampleexporter:
66
processors:
77
exampleprocessor:
8-
pipelines:
9-
traces:
10-
receivers: [examplereceiver]
11-
exporters: [exampleexporter]
12-
processors: [exampleprocessor]
8+
service:
9+
pipelines:
10+
traces:
11+
receivers: [examplereceiver]
12+
exporters: [exampleexporter]
13+
processors: [exampleprocessor]

config/testdata/invalid-exporter-section.yaml

+8-8
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,11 @@ extensions:
1010
service:
1111
extensions:
1212
- exampleextension
13-
pipelines:
14-
traces:
15-
receivers:
16-
- examplereceiver
17-
processors:
18-
- exampleprocessor
19-
exporters:
20-
- exampleexporter
13+
pipelines:
14+
traces:
15+
receivers:
16+
- examplereceiver
17+
processors:
18+
- exampleprocessor
19+
exporters:
20+
- exampleexporter
+7-8
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,15 @@
11
extensions:
22
exampleextension:
3-
service:
4-
extensions: [exampleextension, nosuchextension, and, another, three]
5-
63
receivers:
74
examplereceiver:
85
processors:
96
exampleprocessor:
107
exporters:
118
exampleexporter:
12-
pipelines:
13-
traces:
14-
receivers: [examplereceiver]
15-
processors: [exampleprocessor]
16-
exporters: [exampleexporter]
9+
service:
10+
extensions: [exampleextension, nosuchextension, and, another, three]
11+
pipelines:
12+
traces:
13+
receivers: [examplereceiver]
14+
processors: [exampleprocessor]
15+
exporters: [exampleexporter]

config/testdata/invalid-extension-section.yaml

+8-8
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,11 @@ extensions:
1111
service:
1212
extensions:
1313
- examapleextension
14-
pipelines:
15-
traces:
16-
receivers:
17-
- examplereceiver
18-
processors:
19-
- exampleprocessor
20-
exporters:
21-
- exampleexporter
14+
pipelines:
15+
traces:
16+
receivers:
17+
- examplereceiver
18+
processors:
19+
- exampleprocessor
20+
exporters:
21+
- exampleexporter

config/testdata/invalid-pipeline-section.yaml

+9-9
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,12 @@ extensions:
99
service:
1010
extensions:
1111
- exampleextension
12-
pipelines:
13-
traces:
14-
receivers:
15-
- examplereceiver
16-
processors:
17-
- exampleprocessor
18-
exporters:
19-
- exampleexporter
20-
unknown_section: 1
12+
pipelines:
13+
traces:
14+
receivers:
15+
- examplereceiver
16+
processors:
17+
- exampleprocessor
18+
exporters:
19+
- exampleexporter
20+
unknown_section: 1

config/testdata/invalid-pipeline-type-and-name.yaml

+6-5
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,9 @@ processors:
55
exporters:
66
exampleexporter:
77

8-
pipelines:
9-
/metrics:
10-
receivers: [examplereceiver]
11-
processors: [exampleprocessor]
12-
exporters: [exampleexporter]
8+
service:
9+
pipelines:
10+
/metrics:
11+
receivers: [examplereceiver]
12+
processors: [exampleprocessor]
13+
exporters: [exampleexporter]

0 commit comments

Comments
 (0)