[confmap] Fix a potential race condition between provider and resolver (#14018)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description

A provider CAN use the `watcher` callback if it detects any config
changes. Moreover, it can also be called from different goroutine.
However, resolver's `Shutdown` method closes the `watcher` channel first
and then proceeds to close the providers.

This results in a race condition/panic:
- If the provider detects a new change and attempts to call the
`watcher` callback from another goroutine, which sends the event on an
already closed channel.

This PR reorders the `Shutdown` method to close the providers first and
then close the channel. It also documents that it is provider's
responsibility to close and wait for any goroutines in its `Shutdown`
method

<!-- Issue number if applicable -->
#### Link to tracking issue
Fixes #

<!--Describe what testing was performed and which tests were added.-->
#### Testing

<!--Describe the documentation added.-->
#### Documentation

<!--Please delete paragraphs that you did not use before submitting.-->

---------

Co-authored-by: Alex Boten <[email protected]>

Author: VihasMakwana

.chloggen/fix-ordering-watcher-confmap.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: pkg/confmap
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Fix a potential race condition in confmap by closing the providers first.
+
+# One or more tracking issues or pull requests related to the change
+issues: [14018]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

confmap/provider.go

@@ -86,6 +86,8 @@ type Provider interface {
 	// This method must be called when the Collector service ends, either in case of
 	// success or error. Retrieve cannot be called after Shutdown.
 	//
+	// Provider MUST shutdown and wait for any goroutine(s) that were created to call `watcher`, if any.
+	//
 	// Should never be called concurrently with itself or with Retrieve.
 	// If ctx is cancelled should return immediately with an error.
 	Shutdown(ctx context.Context) error

confmap/resolver.go

@@ -243,14 +243,14 @@ func (mr *Resolver) Watch() <-chan error {
 //
 // Should never be called concurrently with itself or Get.
 func (mr *Resolver) Shutdown(ctx context.Context) error {
-	close(mr.watcher)
-
 	var errs error
 	errs = multierr.Append(errs, mr.closeIfNeeded(ctx))
 	for _, p := range mr.providers {
 		errs = multierr.Append(errs, p.Shutdown(ctx))
 	}
 
+	close(mr.watcher)
+
 	return errs
 }
 

confmap/resolver_test.go

@@ -524,3 +524,47 @@ func newConfFromFile(tb testing.TB, fileName string) map[string]any {
 
 	return NewFromStringMap(data).ToStringMap()
 }
+
+type provider struct {
+	wg sync.WaitGroup
+}
+
+func newRaceDetectorProvider() ProviderFactory {
+	return NewProviderFactory(func(_ ProviderSettings) Provider {
+		return &provider{}
+	})
+}
+
+func (p *provider) Retrieve(_ context.Context, _ string, watcher WatcherFunc) (*Retrieved, error) {
+	p.wg.Add(1)
+	go func() {
+		// mock a config change event and wait for goroutine to return.
+		defer p.wg.Done()
+		watcher(&ChangeEvent{})
+	}()
+	return NewRetrieved(map[string]any{})
+}
+
+func (p *provider) Scheme() string {
+	return "race"
+}
+
+func (p *provider) Shutdown(context.Context) error {
+	p.wg.Wait()
+	return nil
+}
+
+func TestProviderRaceCondition(t *testing.T) {
+	resolver, err := NewResolver(ResolverSettings{
+		URIs: []string{"race:"},
+		ProviderFactories: []ProviderFactory{
+			newRaceDetectorProvider(),
+		},
+		ConverterFactories: nil,
+	})
+	require.NoError(t, err)
+	c, err := resolver.Resolve(context.Background())
+	require.NoError(t, err)
+	require.NotNil(t, c)
+	require.NoError(t, resolver.Shutdown(context.Background()))
+}