[BUG] Add proper locking before modifying the ip to key map (chroma-c…

…ore#1984)

## Description of changes

*Summarize the changes made by this PR.*
 - Improvements & Bug fixes
	 - This PR adds proper locking to the ip to key map. 
 - New functionality
	 - ...

## Test plan
*How are these changes tested?*

- [ ] Tests pass locally with `pytest` for python, `yarn test` for js,
`cargo test` for rust

## Documentation Changes
*Are all docstrings for user-facing APIs updated if required? Do we need
to make documentation changes in the [docs
repository](https://github.com/chroma-core/docs)?*

go/pkg/memberlist_manager/node_watcher.go

@@ -95,7 +95,9 @@ func (w *KubernetesWatcher) Start() error {
 			if err == nil {
 				log.Info("Kubernetes Pod Updated", zap.String("key", key), zap.String("ip", objPod.Status.PodIP))
 				ip := objPod.Status.PodIP
+				w.mu.Lock()
 				w.ipToKey[ip] = key
+				w.mu.Unlock()
 				w.notify(ip)
 			} else {
 				log.Error("Error while getting key from object", zap.Error(err))
@@ -111,7 +113,9 @@ func (w *KubernetesWatcher) Start() error {
 				log.Info("Kubernetes Pod Deleted", zap.String("ip", objPod.Status.PodIP))
 				ip := objPod.Status.PodIP
 				// The contract for GetStatus is that if the ip is not in this map, then it returns NotReady
+				w.mu.Lock()
 				delete(w.ipToKey, ip)
+				w.mu.Unlock()
 				w.notify(ip)
 			} else {
 				log.Error("Error while getting key from object", zap.Error(err))