pool: fix double decrement of hsm requests

Motivation:
If hsm requests completed twice due to a bug or a race condition, then
`AbstractRequest#removeFromQueue` can be called twice. As
removeFromQueue uses `state#get` then two competing requests might get
the same value and introduce double decrement.

Modification:
Add new state 'REMOVED' that with get-and-set semantic avoids double decrement situation.

Result:
Fix double decrement on active hsm requests

Fixes: #7511
Acked-by: Dmitry Litvintsev
Target: master, 10.1, 10.0, 9.2
Require-book: no
Require-notes: yes
(cherry picked from commit daad2ab)
Signed-off-by: Tigran Mkrtchyan <[email protected]>

modules/dcache/src/main/java/org/dcache/pool/nearline/NearlineStorageHandler.java

@@ -1,6 +1,6 @@
 /* dCache - http://www.dcache.org/
  *
- * Copyright (C) 2014 - 2023 Deutsches Elektronen-Synchrotron
+ * Copyright (C) 2014 - 2024 Deutsches Elektronen-Synchrotron
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as
@@ -519,7 +519,7 @@ private void addFromNearlineStorage(StorageInfoMessage message, NearlineStorage
      */
     private abstract static class AbstractRequest<K> implements Comparable<AbstractRequest<K>> {
 
-        protected enum State {QUEUED, ACTIVE, CANCELED}
+        protected enum State {QUEUED, ACTIVE, CANCELED, REMOVED}
 
         private final List<CompletionHandler<Void, K>> callbacks = new ArrayList<>();
         protected final long createdAt = System.currentTimeMillis();
@@ -644,7 +644,7 @@ public int compareTo(AbstractRequest<K> o) {
         }
 
         public void removeFromQueue() {
-            State currentState = state.get();
+            State currentState = state.getAndSet(State.REMOVED);
             switch (currentState) {
                 case QUEUED:
                     decQueued();
@@ -655,6 +655,9 @@ public void removeFromQueue() {
                 case CANCELED:
                     decCanceled();
                     break;
+                case REMOVED:
+                    LOGGER.warn("Request {} was already removed from the queue.", this);
+                    break;
                 default:
                     throw new RuntimeException();
             }