[extension/kafkatopicsobserver] add kafka topics observer extension (#37669)

#### Description
Initial PR for adding kafka topics observer extension. Continuation of
#37372 after renaming a branch.

#### Link to tracking issue
New component
#37665

Co-authored-by: Sean Marciniak <[email protected]>

Author: wojtekzyla

.chloggen/feat_kafka-topics-observer.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: new_component
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: kafkatopicsobserver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Adding new kafka topics observer extension
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [37665]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

.github/CODEOWNERS

@@ -114,6 +114,7 @@ extension/observer/dockerobserver/                               @open-telemetry
 extension/observer/ecsobserver/                                  @open-telemetry/collector-contrib-approvers @dmitryax
 extension/observer/hostobserver/                                 @open-telemetry/collector-contrib-approvers @MovieStoreGuy
 extension/observer/k8sobserver/                                  @open-telemetry/collector-contrib-approvers @dmitryax @ChrsMark
+extension/observer/kafkatopicsobserver/                          @open-telemetry/collector-contrib-approvers @MovieStoreGuy
 extension/oidcauthextension/                                     @open-telemetry/collector-contrib-approvers @jpkrohling
 extension/opampcustommessages/                                   @open-telemetry/collector-contrib-approvers @evan-bradley
 extension/opampextension/                                        @open-telemetry/collector-contrib-approvers @portertech @evan-bradley @tigrannajaryan

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -117,6 +117,7 @@ body:
       - extension/observer/ecstaskobserver
       - extension/observer/hostobserver
       - extension/observer/k8sobserver
+      - extension/observer/kafkatopicsobserver
       - extension/oidcauth
       - extension/opamp
       - extension/opampcustommessages

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -111,6 +111,7 @@ body:
       - extension/observer/ecstaskobserver
       - extension/observer/hostobserver
       - extension/observer/k8sobserver
+      - extension/observer/kafkatopicsobserver
       - extension/oidcauth
       - extension/opamp
       - extension/opampcustommessages

.github/ISSUE_TEMPLATE/other.yaml

@@ -111,6 +111,7 @@ body:
       - extension/observer/ecstaskobserver
       - extension/observer/hostobserver
       - extension/observer/k8sobserver
+      - extension/observer/kafkatopicsobserver
       - extension/oidcauth
       - extension/opamp
       - extension/opampcustommessages

.github/ISSUE_TEMPLATE/unmaintained.yaml

@@ -116,6 +116,7 @@ body:
       - extension/observer/ecstaskobserver
       - extension/observer/hostobserver
       - extension/observer/k8sobserver
+      - extension/observer/kafkatopicsobserver
       - extension/oidcauth
       - extension/opamp
       - extension/opampcustommessages

cmd/otelcontribcol/builder-config.yaml

@@ -167,7 +167,7 @@ receivers:
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/haproxyreceiver v0.119.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.119.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/httpcheckreceiver v0.119.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/huaweicloudcesreceiver v0.119.0  
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/huaweicloudcesreceiver v0.119.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/influxdbreceiver v0.119.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/iisreceiver v0.119.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jaegerreceiver v0.119.0

extension/observer/kafkatopicsobserver/Makefile

@@ -0,0 +1 @@
+include ../../../Makefile.Common

extension/observer/kafkatopicsobserver/README.md

@@ -0,0 +1,60 @@
+# Kafka Topics Observer Extension
+<!-- status autogenerated section -->
+| Status        |           |
+| ------------- |-----------|
+| Stability     | [beta]  |
+| Distributions | [contrib] |
+| Issues        | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Aextension%2Fkafkatopicsobserver%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Aextension%2Fkafkatopicsobserver) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Aextension%2Fkafkatopicsobserver%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Aextension%2Fkafkatopicsobserver) |
+| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner)    | [@MovieStoreGuy](https://www.github.com/MovieStoreGuy) |
+
+[beta]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#beta
+[contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib
+<!-- end autogenerated section -->
+
+The Kafka topics observer extension is a [Receiver Creator](../../../receiver/receivercreator/README.md)-compatible "watch observer" that will detect and report
+kafka topics in kafka cluster based on regex template. This observer watches available topics and matches them with the
+provided regex. If any change in available topics matching the regex is detected, the observer updates the endpoints list.
+
+## Configuration
+
+The following settings are required:
+
+- `protocol_version` (no default): Kafka protocol version e.g. 2.0.0
+
+The following settings can be optionally configured:
+
+- `brokers` (default = localhost:9092): The list of kafka brokers
+- `resolve_canonical_bootstrap_servers_only` (default = false): Whether to resolve then reverse-lookup broker IPs during startup
+- `topic_regex` regex pattern of the topic name to subscribe to.
+- `session_timeout` (default = `10s`): The request timeout for detecting client failures when using Kafka’s group management facilities.
+- `heartbeat_interval` (default = `3s`): The expected time between heartbeats to the consumer coordinator when using Kafka’s group management facilities.
+- `auth`
+    - `plain_text`
+        - `username`: The username to use.
+        - `password`: The password to use
+    - `sasl`
+        - `username`: The username to use.
+        - `password`: The password to use
+        - `mechanism`: The sasl mechanism to use (SCRAM-SHA-256, SCRAM-SHA-512, AWS_MSK_IAM, AWS_MSK_IAM_OAUTHBEARER or PLAIN)
+        - `aws_msk.region`: AWS Region in case of AWS_MSK_IAM or AWS_MSK_IAM_OAUTHBEARER mechanism
+        - `aws_msk.broker_addr`: MSK Broker address in case of AWS_MSK_IAM mechanism
+    - `tls`
+        - `ca_file`: path to the CA cert. For a client this verifies the server certificate. Should
+          only be used if `insecure` is set to false.
+        - `cert_file`: path to the TLS cert to use for TLS required connections. Should
+          only be used if `insecure` is set to false.
+        - `key_file`: path to the TLS key to use for TLS required connections. Should
+          only be used if `insecure` is set to false.
+        - `insecure` (default = false): Disable verifying the server's certificate
+          chain and host name (`InsecureSkipVerify` in the tls config)
+        - `server_name_override`: ServerName indicates the name of the server requested by the client
+          in order to support virtual hosting.
+    - `kerberos`
+        - `service_name`: Kerberos service name
+        - `realm`: Kerberos realm
+        - `use_keytab`: Use of keytab instead of password, if this is true, keytab file will be used instead of password
+        - `username`: The Kerberos username used for authenticate with KDC
+        - `password`: The Kerberos password used for authenticate with KDC
+        - `config_file`: Path to Kerberos configuration. i.e /etc/krb5.conf
+        - `keytab_file`: Path to keytab file. i.e /etc/security/kafka.keytab
+        - `disable_fast_negotiation`: Disable PA-FX-FAST negotiation (Pre-Authentication Framework - Fast). Some common Kerberos implementations do not support PA-FX-FAST negotiation. This is set to `false` by default.

extension/observer/kafkatopicsobserver/config.go

@@ -0,0 +1,33 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package kafkatopicsobserver // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/kafkatopicsobserver"
+
+import (
+	"time"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/kafka"
+)
+
+// Config defines configuration for docker observer
+type Config struct {
+	// The list of kafka brokers (default localhost:9092)
+	Brokers []string `mapstructure:"brokers"`
+	// ResolveCanonicalBootstrapServersOnly makes Sarama do a DNS lookup for
+	// each of the provided brokers. It will then do a PTR lookup for each
+	// returned IP, and that set of names becomes the broker list. This can be
+	// required in SASL environments.
+	ResolveCanonicalBootstrapServersOnly bool `mapstructure:"resolve_canonical_bootstrap_servers_only"`
+	// Kafka protocol version
+	ProtocolVersion string `mapstructure:"protocol_version"`
+	// Session interval for the Kafka consumer
+	SessionTimeout time.Duration `mapstructure:"session_timeout"`
+	// Heartbeat interval for the Kafka consumer
+	HeartbeatInterval time.Duration        `mapstructure:"heartbeat_interval"`
+	Authentication    kafka.Authentication `mapstructure:"auth"`
+	TopicRegex        string               `mapstructure:"topic_regex"`
+}
+
+func (config Config) Validate() error {
+	return nil
+}

extension/observer/kafkatopicsobserver/doc.go

@@ -0,0 +1,6 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+//go:generate mdatagen metadata.yaml
+
+package kafkatopicsobserver // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/kafkatopicsobserver"

extension/observer/kafkatopicsobserver/extension.go

@@ -0,0 +1,58 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package kafkatopicsobserver // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/kafkatopicsobserver"
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/extension"
+	"go.uber.org/zap"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer"
+)
+
+// Mock implementation for the initial PR
+var (
+	_ extension.Extension      = (*kafkaTopicsObserver)(nil)
+	_ observer.EndpointsLister = (*kafkaTopicsObserver)(nil)
+	_ observer.Observable      = (*kafkaTopicsObserver)(nil)
+)
+
+type kafkaTopicsObserver struct {
+	*observer.EndpointsWatcher
+	logger *zap.Logger
+	config *Config
+	cancel func()
+	once   *sync.Once
+	_      context.Context
+}
+
+func newObserver(logger *zap.Logger, config *Config) (extension.Extension, error) {
+	d := &kafkaTopicsObserver{
+		logger: logger, config: config,
+		once: &sync.Once{},
+		cancel: func() {
+		},
+	}
+	d.EndpointsWatcher = observer.NewEndpointsWatcher(d, time.Second, logger)
+	return d, nil
+}
+
+func (d *kafkaTopicsObserver) ListEndpoints() []observer.Endpoint {
+	var endpoints []observer.Endpoint
+	return endpoints
+}
+
+func (d *kafkaTopicsObserver) Start(_ context.Context, _ component.Host) error {
+	return nil
+}
+
+func (d *kafkaTopicsObserver) Shutdown(_ context.Context) error {
+	d.StopListAndWatch()
+	d.cancel()
+	return nil
+}

extension/observer/kafkatopicsobserver/factory.go

@@ -0,0 +1,47 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package kafkatopicsobserver // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/kafkatopicsobserver"
+
+import (
+	"context"
+	"time"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/extension"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/kafkatopicsobserver/internal/metadata"
+)
+
+const (
+	defaultBroker            = "localhost:9092"
+	defaultSessionTimeout    = 10 * time.Second
+	defaultHeartbeatInterval = 3 * time.Second
+)
+
+// NewFactory should be called to create a factory with default values.
+func NewFactory() extension.Factory {
+	return extension.NewFactory(
+		metadata.Type,
+		createDefaultConfig,
+		createExtension,
+		component.StabilityLevelBeta,
+	)
+}
+
+func createDefaultConfig() component.Config {
+	return &Config{
+		Brokers:           []string{defaultBroker},
+		SessionTimeout:    defaultSessionTimeout,
+		HeartbeatInterval: defaultHeartbeatInterval,
+	}
+}
+
+func createExtension(
+	_ context.Context,
+	settings extension.Settings,
+	cfg component.Config,
+) (extension.Extension, error) {
+	config := cfg.(*Config)
+	return newObserver(settings.Logger, config)
+}