Merge pull request #2 from wcmc-its/retry_impl

Implement retry of scopus api based on non 200 response
wcmc-its · Apr 1, 2019 · 22b5b57 · 22b5b57
2 parents a02a1f4 + 057ed54
commit 22b5b57
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 4 deletions.
diff --git a/pom.xml b/pom.xml
@@ -53,6 +53,11 @@
         		</exclusion>
         	</exclusions>
         </dependency>
+        <dependency>
+      		<groupId>com.github.rholder</groupId>
+      		<artifactId>guava-retrying</artifactId>
+      		<version>2.0.0</version>
+    	</dependency>
     </dependencies>
 
     <build>

diff --git a/src/main/java/reciter/scopus/retriever/ScopusArticleRetriever.java b/src/main/java/reciter/scopus/retriever/ScopusArticleRetriever.java
@@ -1,15 +1,23 @@
 package reciter.scopus.retriever;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.github.rholder.retry.Retryer;
+import com.github.rholder.retry.Retryer.RetryerCallable;
+import com.github.rholder.retry.RetryerBuilder;
+import com.github.rholder.retry.StopStrategies;
+import com.github.rholder.retry.WaitStrategies;
+import com.google.common.base.Predicates;
+
 import reciter.model.scopus.ScopusArticle;
 import reciter.scopus.callable.ScopusUriParserCallable;
 import reciter.scopus.querybuilder.ScopusXmlQuery;
@@ -29,6 +37,11 @@ public class ScopusArticleRetriever {
 	 */
 	protected static final int SCOPUS_MAX_THRESHOLD = 25;
 
+	/**
+	 * @param pmids scopus query could be DOI, scopusDocId, PMID
+	 * @param type to query scopus like DOI(), SCOPUS-ID(), PMID() etc.
+	 * @return
+	 */
 	public List<ScopusArticle> retrieveScopus(List<Object> pmids, String type) {
 		slf4jLogger.info("Pmids:" + pmids);
 		List<String> pmidQueries = new ArrayList<>();
@@ -62,16 +75,25 @@ public List<ScopusArticle> retrieveScopus(List<Object> pmids, String type) {
 			}
 		}
 
-		List<Callable<List<ScopusArticle>>> callables = new ArrayList<>();
+		List<RetryerCallable<List<ScopusArticle>>> callables = new ArrayList<RetryerCallable<List<ScopusArticle>>>();
+		Retryer<List<ScopusArticle>> retryer = RetryerBuilder.<List<ScopusArticle>>newBuilder()
+                .retryIfResult(Predicates.<List<ScopusArticle>>isNull())
+                .retryIfExceptionOfType(IOException.class)
+                .retryIfRuntimeException()
+                .withWaitStrategy(WaitStrategies.fibonacciWait(100L, 2L, TimeUnit.MINUTES))
+                .withStopStrategy(StopStrategies.stopAfterAttempt(10))
+                .build();
 
 		for (String query : pmidQueries) {
 			ScopusXmlQuery scopusXmlQuery = new ScopusXmlQuery.ScopusXmlQueryBuilder(query, SCOPUS_MAX_THRESHOLD).build();
 			String scopusUrl = scopusXmlQuery.getQueryUrl();
 			ScopusUriParserCallable scopusUriParserCallable = new ScopusUriParserCallable(new ScopusXmlHandler(), scopusUrl);
-			callables.add(scopusUriParserCallable);
+			RetryerCallable<List<ScopusArticle>> retryerCallable = retryer.wrap(scopusUriParserCallable);
+        	callables.add(retryerCallable);
+			//callables.add(scopusUriParserCallable);
 		}
 
-		List<List<ScopusArticle>> list = new ArrayList<>();
+		List<List<ScopusArticle>> list = new ArrayList<List<ScopusArticle>>();
 
 		int numAvailableProcessors = Runtime.getRuntime().availableProcessors();
 		ExecutorService executor = Executors.newFixedThreadPool(numAvailableProcessors);