From 1c0291ae44ed10bc0cddb9121820ea7c6070df7e Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Wed, 17 Aug 2022 12:23:38 -0400 Subject: [PATCH 01/12] Implement batchRefresh logic --- .../core/command/AbstractConfigCommand.java | 112 ++++++++++++------ .../java/gyro/core/resource/Resource.java | 14 +++ 2 files changed, 88 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java index b6a9358e3..ea1bada1e 100644 --- a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java +++ b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java @@ -22,6 +22,8 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -205,6 +207,7 @@ private void refreshResources(RootScope scope) { ExecutorService refreshService = Executors.newWorkStealingPool(25); List refreshes = new ArrayList<>(); + Map> refreshQueues = new HashMap<>(); for (FileScope fileScope : scope.getFileScopes()) { String currentDirectory = System.getProperty("user.dir"); @@ -226,84 +229,117 @@ private void refreshResources(RootScope scope) { processors.addAll(0, s.getSettings(ChangeSettings.class).getProcessors()); } - refreshes.add(new Refresh(resource, ui, refreshService.submit(() -> { - GyroCore.pushUi(ui); + List refreshQueue = + refreshQueues.computeIfAbsent(DiffableType.getInstance(resource), m -> new ArrayList<>()); + refreshQueue.add(resource); + } + } + + for (DiffableType type : refreshQueues.keySet()) { + int count = refreshQueues.get(type).size(); + ui.write("Refreshing @|magenta,bold %s|@ resources: @|green %s|@\n", type.getName(), count); + + List refreshQueue = refreshQueues.get(type); + + refreshes.add(new Refresh(refreshQueue, ui, refreshService.submit(() -> { + GyroCore.pushUi(ui); + + if (refreshQueue.isEmpty()) { + return false; + } + + started.incrementAndGet(); - started.incrementAndGet(); + // Run beforeRefresh processors + for (Resource resource : refreshQueue) { + List processors = new ArrayList<>(); + for (Scope s = DiffableInternals.getScope(resource); s != null; s = s.getParent()) { + processors.addAll(0, s.getSettings(ChangeSettings.class).getProcessors()); + } for (ChangeProcessor processor : processors) { processor.beforeRefresh(ui, resource); } + } - boolean keep = resource.refresh(); + Resource peek = refreshQueue.get(0); + Map refreshResults = peek.batchRefresh(refreshQueue); + + // Run afterRefresh processors + for (Resource resource : refreshQueue) { + List processors = new ArrayList<>(); + for (Scope s = DiffableInternals.getScope(resource); s != null; s = s.getParent()) { + processors.addAll(0, s.getSettings(ChangeSettings.class).getProcessors()); + } for (ChangeProcessor processor : processors) { processor.afterRefresh(ui, resource); } - if (keep) { + if (refreshResults.containsKey(resource) && refreshResults.get(resource)) { DiffableInternals.getModifications(resource).forEach(m -> m.refresh(resource)); - } - - done.incrementAndGet(); - - if (keep) { DiffableInternals.disconnect(resource, true); DiffableInternals.update(resource); - return false; - - } else { - return true; } - }))); - } + } + + done.incrementAndGet(); + + return true; + }))); } refreshService.shutdown(); - for (Refresh refresh : refreshes) { - Resource resource = refresh.resource; - String typeName = DiffableType.getInstance(resource).getName(); - String name = DiffableInternals.getName(resource); + int refreshCount = 0; + for (Refresh refresh : refreshes) { try { - if (refresh.future.get()) { + refresh.future.get(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (ExecutionException e) { + throw new RuntimeException(e); + } + + for (Resource resource : refresh.resources) { + String typeName = DiffableType.getInstance(resource).getName(); + String name = DiffableInternals.getName(resource); + + if (refresh.isRefreshed(resource)) { ui.replace("@|magenta - Removing from state:|@ %s %s\n", typeName, name); scope.getFileScopes().forEach(s -> s.remove(resource.primaryKey())); } - } catch (ExecutionException error) { - refreshService.shutdownNow(); - messageService.shutdown(); - - ui.write("\n"); - - throw new GyroException( - String.format("Can't refresh @|bold %s %s|@ resource!", typeName, name), - error.getCause()); - - } catch (InterruptedException error) { - Thread.currentThread().interrupt(); - return; + refreshCount++; } } messageService.shutdown(); - ui.replace("@|magenta ⟳ Refreshed resources:|@ %s\n", refreshes.size()); + ui.replace("@|magenta ⟳ Refreshed resources:|@ %s\n", refreshCount); } private static class Refresh { - public final Resource resource; + public final List resources; + public final Set refreshed; public final Future future; public final GyroUI ui; - public Refresh(Resource resource, GyroUI ui, Future future) { - this.resource = resource; + public Refresh(List resources, GyroUI ui, Future future) { + this.resources = resources; this.ui = ui; this.future = future; + this.refreshed = new HashSet<>(); } + public boolean isRefreshed(Resource resource) { + return refreshed.contains(resource); + } + + public void markRefreshed(Resource resource) { + refreshed.add(resource); + } } } diff --git a/core/src/main/java/gyro/core/resource/Resource.java b/core/src/main/java/gyro/core/resource/Resource.java index 3d7fd200f..a966fe324 100644 --- a/core/src/main/java/gyro/core/resource/Resource.java +++ b/core/src/main/java/gyro/core/resource/Resource.java @@ -16,6 +16,9 @@ package gyro.core.resource; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; @@ -27,6 +30,17 @@ public abstract class Resource extends Diffable { public abstract boolean refresh(); + public Map batchRefresh(List resources) { + Map refreshResults = new HashMap<>(); + + for (Resource resource : resources) { + boolean keep = resource.refresh(); + refreshResults.put(resource, keep); + } + + return refreshResults; + } + public abstract void create(GyroUI ui, State state) throws Exception; public abstract void update(GyroUI ui, State state, Resource current, Set changedFieldNames) From be8b9f8e6878d4eff28ac0dc4e6137d30614d3b4 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Wed, 17 Aug 2022 15:30:42 -0400 Subject: [PATCH 02/12] Remove ElapsedTimeChangeProcessor --- .../resource/ElapsedTimeChangeProcessor.java | 88 ------------------- .../main/java/gyro/core/scope/RootScope.java | 4 +- 2 files changed, 1 insertion(+), 91 deletions(-) delete mode 100644 core/src/main/java/gyro/core/resource/ElapsedTimeChangeProcessor.java diff --git a/core/src/main/java/gyro/core/resource/ElapsedTimeChangeProcessor.java b/core/src/main/java/gyro/core/resource/ElapsedTimeChangeProcessor.java deleted file mode 100644 index 8464cc226..000000000 --- a/core/src/main/java/gyro/core/resource/ElapsedTimeChangeProcessor.java +++ /dev/null @@ -1,88 +0,0 @@ -package gyro.core.resource; - -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import gyro.core.GyroUI; -import gyro.core.diff.ChangeProcessor; -import gyro.core.scope.State; -import org.apache.commons.lang.time.StopWatch; - -public class ElapsedTimeChangeProcessor extends ChangeProcessor { - - private Map stopWatches = new HashMap<>(); - private static final String ELAPSED_TIME_MESSAGE = " (elapsed time: @|green %s|@) "; - - private synchronized StopWatch start(Resource resource) { - StopWatch stopWatch = stopWatches.computeIfAbsent(resource, r -> new StopWatch()); - stopWatch.reset(); - stopWatch.start(); - - return stopWatch; - } - - private synchronized String stop(GyroUI ui, Resource resource) { - StopWatch stopWatch = stopWatches.get(resource); - if (stopWatch == null) { - return "0ms"; - } - - stopWatch.stop(); - - Duration duration = Duration.ofMillis(stopWatch.getTime()); - - if (duration.getSeconds() <= 10) { - return String.format("%dms", duration.toMillis()); - } else { - return String.format("%dm%ds", duration.toMinutes(), (duration.getSeconds() - (duration.toMinutes() * 60))); - } - } - - @Override - public void beforeRefresh(GyroUI ui, Resource resource) throws Exception { - start(resource); - } - - @Override - public void afterRefresh(GyroUI ui, Resource resource) throws Exception { - String elapsed = stop(ui, resource); - ui.write("Refreshing @|magenta,bold %s|@ took: @|green %s|@\n", resource.primaryKey(), elapsed); - } - - @Override - public void beforeCreate(GyroUI ui, State state, Resource resource) throws Exception { - start(resource); - } - - @Override - public void afterCreate(GyroUI ui, State state, Resource resource) throws Exception { - String elapsed = stop(ui, resource); - ui.write(ELAPSED_TIME_MESSAGE, elapsed); - } - - @Override - public void beforeUpdate( - GyroUI ui, State state, Resource current, Resource pending, Set changedFields) throws Exception { - start(pending); - } - - @Override - public void afterUpdate( - GyroUI ui, State state, Resource current, Resource pending, Set changedFields) throws Exception { - String elapsed = stop(ui, current); - ui.write(ELAPSED_TIME_MESSAGE, elapsed); - } - - @Override - public void beforeDelete(GyroUI ui, State state, Resource resource) throws Exception { - start(resource); - } - - @Override - public void afterDelete(GyroUI ui, State state, Resource resource) throws Exception { - String elapsed = stop(ui, resource); - ui.write(ELAPSED_TIME_MESSAGE, elapsed); - } -} diff --git a/core/src/main/java/gyro/core/scope/RootScope.java b/core/src/main/java/gyro/core/scope/RootScope.java index 0838b318c..bd5a86806 100644 --- a/core/src/main/java/gyro/core/scope/RootScope.java +++ b/core/src/main/java/gyro/core/scope/RootScope.java @@ -79,7 +79,6 @@ import gyro.core.resource.DiffableField; import gyro.core.resource.DiffableInternals; import gyro.core.resource.DiffableType; -import gyro.core.resource.ElapsedTimeChangeProcessor; import gyro.core.resource.ExtendsDirectiveProcessor; import gyro.core.resource.ModificationChangeProcessor; import gyro.core.resource.ModificationPlugin; @@ -179,8 +178,7 @@ public RootScope( Stream.of( new ModificationChangeProcessor(), - new ConfiguredFieldsChangeProcessor(), - new ElapsedTimeChangeProcessor()) + new ConfiguredFieldsChangeProcessor()) .forEach(p -> getSettings(ChangeSettings.class).getProcessors().add(p)); Stream.of( From 22edd6a824df45f15e5885bd2551ec749bd943b7 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Wed, 17 Aug 2022 15:31:16 -0400 Subject: [PATCH 03/12] Output elapsed time for groups of resources --- .../core/command/AbstractConfigCommand.java | 46 +++++++++++++++---- .../java/gyro/core/resource/Resource.java | 31 ++++++++++++- 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java index ea1bada1e..5a2a2b829 100644 --- a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java +++ b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java @@ -20,6 +20,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.text.SimpleDateFormat; +import java.time.Duration; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -56,6 +57,7 @@ import gyro.core.scope.RootScope; import gyro.core.scope.Scope; import gyro.core.scope.State; +import org.apache.commons.lang.time.StopWatch; import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; @@ -236,16 +238,13 @@ private void refreshResources(RootScope scope) { } for (DiffableType type : refreshQueues.keySet()) { - int count = refreshQueues.get(type).size(); - ui.write("Refreshing @|magenta,bold %s|@ resources: @|green %s|@\n", type.getName(), count); - List refreshQueue = refreshQueues.get(type); refreshes.add(new Refresh(refreshQueue, ui, refreshService.submit(() -> { GyroCore.pushUi(ui); if (refreshQueue.isEmpty()) { - return false; + return 0L; } started.incrementAndGet(); @@ -263,7 +262,10 @@ private void refreshResources(RootScope scope) { } Resource peek = refreshQueue.get(0); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); Map refreshResults = peek.batchRefresh(refreshQueue); + stopWatch.stop(); // Run afterRefresh processors for (Resource resource : refreshQueue) { @@ -285,7 +287,7 @@ private void refreshResources(RootScope scope) { done.incrementAndGet(); - return true; + return stopWatch.getTime(); }))); } @@ -294,14 +296,30 @@ private void refreshResources(RootScope scope) { int refreshCount = 0; for (Refresh refresh : refreshes) { + long refreshDuration = 0; try { - refresh.future.get(); + refreshDuration = refresh.future.get(); } catch (InterruptedException e) { throw new RuntimeException(e); } catch (ExecutionException e) { throw new RuntimeException(e); } + Duration duration = Duration.ofMillis(refreshDuration); + + String time = ""; + if (duration.getSeconds() <= 10) { + time = String.format("%dms", duration.toMillis()); + } else { + time = String.format("%dm%ds", duration.toMinutes(), (duration.getSeconds() - (duration.toMinutes() * 60))); + } + + ui.write("Refreshing @|magenta,bold %s|@: @|green %s|@ %s refreshed in @|green %s|@ elapsed\n", + refresh.diffableTypeName(), + refresh.count(), + refresh.count() == 1 ? "resource" : "resources", + time); + for (Resource resource : refresh.resources) { String typeName = DiffableType.getInstance(resource).getName(); String name = DiffableInternals.getName(resource); @@ -323,16 +341,28 @@ private static class Refresh { public final List resources; public final Set refreshed; - public final Future future; + public final Future future; public final GyroUI ui; - public Refresh(List resources, GyroUI ui, Future future) { + public Refresh(List resources, GyroUI ui, Future future) { this.resources = resources; this.ui = ui; this.future = future; this.refreshed = new HashSet<>(); } + public long count() { + return resources.size(); + } + + public String diffableTypeName() { + if (resources.isEmpty()) { + return "unknown"; + } + + return DiffableType.getInstance(resources.get(0)).getName(); + } + public boolean isRefreshed(Resource resource) { return refreshed.contains(resource); } diff --git a/core/src/main/java/gyro/core/resource/Resource.java b/core/src/main/java/gyro/core/resource/Resource.java index a966fe324..64830b8f0 100644 --- a/core/src/main/java/gyro/core/resource/Resource.java +++ b/core/src/main/java/gyro/core/resource/Resource.java @@ -16,11 +16,16 @@ package gyro.core.resource; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import gyro.core.GyroUI; import gyro.core.auth.Credentials; @@ -33,9 +38,20 @@ public abstract class Resource extends Diffable { public Map batchRefresh(List resources) { Map refreshResults = new HashMap<>(); + ExecutorService refreshService = Executors.newWorkStealingPool(24); + List refreshes = new ArrayList<>(); + for (Resource resource : resources) { - boolean keep = resource.refresh(); - refreshResults.put(resource, keep); + refreshes.add(new Refresh(resource, refreshService.submit(() -> resource.refresh()))); + } + + try { + for (Refresh refresh : refreshes) { + refreshResults.put(refresh.resource, refresh.future.get()); + } + } catch (InterruptedException | ExecutionException e) { + refreshService.shutdown(); + throw new RuntimeException(e); } return refreshResults; @@ -66,4 +82,15 @@ public Object get(String key) { public String primaryKey() { return String.format("%s::%s", DiffableType.getInstance(getClass()).getName(), name); } + + private static class Refresh { + + public final Resource resource; + public final Future future; + + public Refresh(Resource resource, Future future) { + this.resource = resource; + this.future = future; + } + } } From 03b8c8dc5d8e5679a4157a7dda7ebfbd86b70e9d Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 18 Aug 2022 07:48:42 -0400 Subject: [PATCH 04/12] Set work pool size based on processors --- core/src/main/java/gyro/core/command/AbstractConfigCommand.java | 2 +- core/src/main/java/gyro/core/resource/Resource.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java index 5a2a2b829..db60a12fa 100644 --- a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java +++ b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java @@ -207,7 +207,7 @@ private void refreshResources(RootScope scope) { ui.replace("@|magenta ⟳ Refreshing resources:|@ %s started, %s done", started.get(), done.get()); }, 0, 100, TimeUnit.MILLISECONDS); - ExecutorService refreshService = Executors.newWorkStealingPool(25); + ExecutorService refreshService = Executors.newWorkStealingPool(); List refreshes = new ArrayList<>(); Map> refreshQueues = new HashMap<>(); diff --git a/core/src/main/java/gyro/core/resource/Resource.java b/core/src/main/java/gyro/core/resource/Resource.java index 64830b8f0..8555f8110 100644 --- a/core/src/main/java/gyro/core/resource/Resource.java +++ b/core/src/main/java/gyro/core/resource/Resource.java @@ -38,7 +38,7 @@ public abstract class Resource extends Diffable { public Map batchRefresh(List resources) { Map refreshResults = new HashMap<>(); - ExecutorService refreshService = Executors.newWorkStealingPool(24); + ExecutorService refreshService = Executors.newWorkStealingPool(); List refreshes = new ArrayList<>(); for (Resource resource : resources) { From b4d7ed9146c4053c01e8c8591c551520615247d5 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 18 Aug 2022 07:59:27 -0400 Subject: [PATCH 05/12] Remove messageService as it's unused --- .../java/gyro/core/command/AbstractConfigCommand.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java index db60a12fa..ba3f31ce9 100644 --- a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java +++ b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java @@ -35,8 +35,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -198,15 +196,10 @@ protected void doExecute() throws Exception { } private void refreshResources(RootScope scope) { - ScheduledExecutorService messageService = Executors.newSingleThreadScheduledExecutor(); GyroUI ui = GyroCore.ui(); AtomicInteger started = new AtomicInteger(); AtomicInteger done = new AtomicInteger(); - messageService.scheduleAtFixedRate(() -> { - ui.replace("@|magenta ⟳ Refreshing resources:|@ %s started, %s done", started.get(), done.get()); - }, 0, 100, TimeUnit.MILLISECONDS); - ExecutorService refreshService = Executors.newWorkStealingPool(); List refreshes = new ArrayList<>(); Map> refreshQueues = new HashMap<>(); @@ -333,7 +326,6 @@ private void refreshResources(RootScope scope) { } } - messageService.shutdown(); ui.replace("@|magenta ⟳ Refreshed resources:|@ %s\n", refreshCount); } From 29addded89eb6b004b4b585767fb6a17a3b266ab Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 18 Aug 2022 09:00:53 -0400 Subject: [PATCH 06/12] Move executor shutdown to finally block --- core/src/main/java/gyro/core/resource/Resource.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/gyro/core/resource/Resource.java b/core/src/main/java/gyro/core/resource/Resource.java index 8555f8110..718b21422 100644 --- a/core/src/main/java/gyro/core/resource/Resource.java +++ b/core/src/main/java/gyro/core/resource/Resource.java @@ -50,8 +50,9 @@ public Map batchRefresh(List re refreshResults.put(refresh.resource, refresh.future.get()); } } catch (InterruptedException | ExecutionException e) { - refreshService.shutdown(); throw new RuntimeException(e); + } finally { + refreshService.shutdown(); } return refreshResults; From 1931c5fdc9b3759335637adb6bcb89dd43c0aabe Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 18 Aug 2022 09:10:06 -0400 Subject: [PATCH 07/12] Cleanup --- .../main/java/gyro/core/command/AbstractConfigCommand.java | 4 +--- core/src/main/java/gyro/core/resource/Resource.java | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java index ba3f31ce9..c9867361a 100644 --- a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java +++ b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java @@ -292,9 +292,7 @@ private void refreshResources(RootScope scope) { long refreshDuration = 0; try { refreshDuration = refresh.future.get(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } catch (ExecutionException e) { + } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } diff --git a/core/src/main/java/gyro/core/resource/Resource.java b/core/src/main/java/gyro/core/resource/Resource.java index 718b21422..b7505c0b2 100644 --- a/core/src/main/java/gyro/core/resource/Resource.java +++ b/core/src/main/java/gyro/core/resource/Resource.java @@ -42,7 +42,7 @@ public Map batchRefresh(List re List refreshes = new ArrayList<>(); for (Resource resource : resources) { - refreshes.add(new Refresh(resource, refreshService.submit(() -> resource.refresh()))); + refreshes.add(new Refresh(resource, refreshService.submit(resource::refresh))); } try { From 9d937d35f7c82a479d0cda7b0ed1a47b7b287d90 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 19 Aug 2022 07:49:43 -0400 Subject: [PATCH 08/12] Refactor refresh to remove unused code This also fixes a bug that prevented deleted resources from being detected. --- .../core/command/AbstractConfigCommand.java | 105 ++++++------------ 1 file changed, 32 insertions(+), 73 deletions(-) diff --git a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java index c9867361a..10d894e94 100644 --- a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java +++ b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java @@ -24,18 +24,15 @@ import java.util.ArrayList; import java.util.Date; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import gyro.core.GyroCore; @@ -197,13 +194,12 @@ protected void doExecute() throws Exception { private void refreshResources(RootScope scope) { GyroUI ui = GyroCore.ui(); - AtomicInteger started = new AtomicInteger(); - AtomicInteger done = new AtomicInteger(); ExecutorService refreshService = Executors.newWorkStealingPool(); List refreshes = new ArrayList<>(); Map> refreshQueues = new HashMap<>(); + // Group Resources by type. for (FileScope fileScope : scope.getFileScopes()) { String currentDirectory = System.getProperty("user.dir"); String currentFile = GyroCore.getRootDirectory().resolve(fileScope.getFile()).getParent().toString(); @@ -230,6 +226,7 @@ private void refreshResources(RootScope scope) { } } + // Refresh each type as a group. for (DiffableType type : refreshQueues.keySet()) { List refreshQueue = refreshQueues.get(type); @@ -237,10 +234,12 @@ private void refreshResources(RootScope scope) { GyroCore.pushUi(ui); if (refreshQueue.isEmpty()) { - return 0L; + return null; } - started.incrementAndGet(); + Resource peek = refreshQueue.get(0); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); // Run beforeRefresh processors for (Resource resource : refreshQueue) { @@ -254,11 +253,7 @@ private void refreshResources(RootScope scope) { } } - Resource peek = refreshQueue.get(0); - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); Map refreshResults = peek.batchRefresh(refreshQueue); - stopWatch.stop(); // Run afterRefresh processors for (Resource resource : refreshQueue) { @@ -278,87 +273,51 @@ private void refreshResources(RootScope scope) { } } - done.incrementAndGet(); - - return stopWatch.getTime(); - }))); - } - - refreshService.shutdown(); - - int refreshCount = 0; - - for (Refresh refresh : refreshes) { - long refreshDuration = 0; - try { - refreshDuration = refresh.future.get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } + stopWatch.stop(); + Duration duration = Duration.ofMillis(stopWatch.getTime()); - Duration duration = Duration.ofMillis(refreshDuration); + String time = ""; + if (duration.getSeconds() <= 10) { + time = String.format("%dms", duration.toMillis()); + } else { + time = String.format("%dm%ds", duration.toMinutes(), (duration.getSeconds() - (duration.toMinutes() * 60))); + } - String time = ""; - if (duration.getSeconds() <= 10) { - time = String.format("%dms", duration.toMillis()); - } else { - time = String.format("%dm%ds", duration.toMinutes(), (duration.getSeconds() - (duration.toMinutes() * 60))); - } + ui.write("Refreshing @|magenta,bold %s|@: @|green %s|@ %s refreshed in @|green %s|@ elapsed\n", + DiffableType.getInstance(peek).getName(), + refreshResults.size(), + refreshResults.size() == 1 ? "resource" : "resources", + time); - ui.write("Refreshing @|magenta,bold %s|@: @|green %s|@ %s refreshed in @|green %s|@ elapsed\n", - refresh.diffableTypeName(), - refresh.count(), - refresh.count() == 1 ? "resource" : "resources", - time); + for (Resource resource : refreshResults.keySet()) { + boolean refreshed = refreshResults.get(resource); - for (Resource resource : refresh.resources) { - String typeName = DiffableType.getInstance(resource).getName(); - String name = DiffableInternals.getName(resource); + String typeName = DiffableType.getInstance(resource).getName(); + String name = DiffableInternals.getName(resource); - if (refresh.isRefreshed(resource)) { - ui.replace("@|magenta - Removing from state:|@ %s %s\n", typeName, name); - scope.getFileScopes().forEach(s -> s.remove(resource.primaryKey())); + if (!refreshed) { + ui.replace("@|magenta - Removing from state:|@ %s %s\n", typeName, name); + scope.getFileScopes().forEach(s -> s.remove(resource.primaryKey())); + } } - refreshCount++; - } + return null; + }))); } - ui.replace("@|magenta ⟳ Refreshed resources:|@ %s\n", refreshCount); + refreshService.shutdown(); } private static class Refresh { public final List resources; - public final Set refreshed; - public final Future future; + public final Future future; public final GyroUI ui; - public Refresh(List resources, GyroUI ui, Future future) { + public Refresh(List resources, GyroUI ui, Future future) { this.resources = resources; this.ui = ui; this.future = future; - this.refreshed = new HashSet<>(); - } - - public long count() { - return resources.size(); - } - - public String diffableTypeName() { - if (resources.isEmpty()) { - return "unknown"; - } - - return DiffableType.getInstance(resources.get(0)).getName(); - } - - public boolean isRefreshed(Resource resource) { - return refreshed.contains(resource); - } - - public void markRefreshed(Resource resource) { - refreshed.add(resource); } } From 41e5c026fa7d2b0c7e313918615022b941c25049 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 19 Aug 2022 11:02:46 -0400 Subject: [PATCH 09/12] Wait for batch refresh to finish --- .../java/gyro/core/command/AbstractConfigCommand.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java index 10d894e94..59f7ee415 100644 --- a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java +++ b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java @@ -30,6 +30,7 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -303,6 +304,14 @@ private void refreshResources(RootScope scope) { return null; }))); + + for (Refresh refresh : refreshes) { + try { + refresh.future.get(); + } catch (InterruptedException | ExecutionException e) { + throw new GyroException(e); + } + } } refreshService.shutdown(); From 151b9dc716d30f633868838a1fd8fbf6d984bfee Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Tue, 23 Aug 2022 11:05:34 -0400 Subject: [PATCH 10/12] Improve errors when batch refresh fails --- .../core/command/AbstractConfigCommand.java | 31 ++++++++++++++- .../gyro/core/resource/RefreshException.java | 39 +++++++++++++++++++ .../java/gyro/core/resource/Resource.java | 14 ++++--- 3 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 core/src/main/java/gyro/core/resource/RefreshException.java diff --git a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java index 59f7ee415..5bc880e28 100644 --- a/core/src/main/java/gyro/core/command/AbstractConfigCommand.java +++ b/core/src/main/java/gyro/core/command/AbstractConfigCommand.java @@ -48,6 +48,7 @@ import gyro.core.diff.ChangeSettings; import gyro.core.resource.DiffableInternals; import gyro.core.resource.DiffableType; +import gyro.core.resource.RefreshException; import gyro.core.resource.Resource; import gyro.core.scope.FileScope; import gyro.core.scope.RootScope; @@ -308,8 +309,29 @@ private void refreshResources(RootScope scope) { for (Refresh refresh : refreshes) { try { refresh.future.get(); - } catch (InterruptedException | ExecutionException e) { - throw new GyroException(e); + } catch (InterruptedException erorr) { + Thread.currentThread().interrupt(); + return; + } catch (ExecutionException error) { + Throwable cause = error.getCause(); + if (cause instanceof RefreshException) { + ui.write("\n"); + + RefreshException refreshException = (RefreshException) cause; + + Resource resource = refreshException.getResource(); + String typeName = DiffableType.getInstance(resource).getName(); + String name = DiffableInternals.getName(resource); + + throw new GyroException( + String.format("Can't refresh @|bold %s %s|@ resource!", typeName, name), + error.getCause()); + } else { + + throw new GyroException( + String.format("Can't refresh @|bold %s |@ resource group!", refresh.typeName()), + error.getCause()); + } } } } @@ -328,6 +350,11 @@ public Refresh(List resources, GyroUI ui, Future future) { this.ui = ui; this.future = future; } + + public String typeName() { + Resource resource = resources.get(0); + return DiffableType.getInstance(resource).getName(); + } } } diff --git a/core/src/main/java/gyro/core/resource/RefreshException.java b/core/src/main/java/gyro/core/resource/RefreshException.java new file mode 100644 index 000000000..57ab2d408 --- /dev/null +++ b/core/src/main/java/gyro/core/resource/RefreshException.java @@ -0,0 +1,39 @@ +package gyro.core.resource; + +public class RefreshException extends RuntimeException { + + private Resource resource; + + public RefreshException(Resource resource) { + this.resource = resource; + } + + public RefreshException(String message, Resource resource) { + super(message); + this.resource = resource; + } + + public RefreshException(String message, Throwable cause, Resource resource) { + super(message, cause); + this.resource = resource; + } + + public RefreshException(Throwable cause, Resource resource) { + super(cause); + this.resource = resource; + } + + public RefreshException( + String message, + Throwable cause, + boolean enableSuppression, + boolean writableStackTrace, + Resource resource) { + super(message, cause, enableSuppression, writableStackTrace); + this.resource = resource; + } + + public Resource getResource() { + return resource; + } +} diff --git a/core/src/main/java/gyro/core/resource/Resource.java b/core/src/main/java/gyro/core/resource/Resource.java index b7505c0b2..bab01bd58 100644 --- a/core/src/main/java/gyro/core/resource/Resource.java +++ b/core/src/main/java/gyro/core/resource/Resource.java @@ -45,14 +45,16 @@ public Map batchRefresh(List re refreshes.add(new Refresh(resource, refreshService.submit(resource::refresh))); } - try { - for (Refresh refresh : refreshes) { + for (Refresh refresh : refreshes) { + try { refreshResults.put(refresh.resource, refresh.future.get()); + } catch (ExecutionException error) { + throw new RefreshException(error, refresh.resource); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } finally { + refreshService.shutdown(); } - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } finally { - refreshService.shutdown(); } return refreshResults; From 9bfba754cb8e96f24a9ca88974ffaa739c8c8aa7 Mon Sep 17 00:00:00 2001 From: Deepanjan Bhattacharyya Date: Tue, 6 Sep 2022 13:36:10 -0700 Subject: [PATCH 11/12] fix display when creating/updating resources re introduce elapsed time change processor skipping the refresh portion to fix the display issue --- .../resource/ElapsedTimeChangeProcessor.java | 77 +++++++++++++++++++ .../main/java/gyro/core/scope/RootScope.java | 2 + 2 files changed, 79 insertions(+) create mode 100644 core/src/main/java/gyro/core/resource/ElapsedTimeChangeProcessor.java diff --git a/core/src/main/java/gyro/core/resource/ElapsedTimeChangeProcessor.java b/core/src/main/java/gyro/core/resource/ElapsedTimeChangeProcessor.java new file mode 100644 index 000000000..c9cdb09f6 --- /dev/null +++ b/core/src/main/java/gyro/core/resource/ElapsedTimeChangeProcessor.java @@ -0,0 +1,77 @@ +package gyro.core.resource; + +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import gyro.core.GyroUI; +import gyro.core.diff.ChangeProcessor; +import gyro.core.scope.State; +import org.apache.commons.lang.time.StopWatch; + +public class ElapsedTimeChangeProcessor extends ChangeProcessor { + + private Map stopWatches = new HashMap<>(); + private static final String ELAPSED_TIME_MESSAGE = " (elapsed time: @|green %s|@) "; + + private synchronized StopWatch start(Resource resource) { + StopWatch stopWatch = stopWatches.computeIfAbsent(resource, r -> new StopWatch()); + stopWatch.reset(); + stopWatch.start(); + + return stopWatch; + } + + private synchronized String stop(GyroUI ui, Resource resource) { + StopWatch stopWatch = stopWatches.get(resource); + if (stopWatch == null) { + return "0ms"; + } + + stopWatch.stop(); + + Duration duration = Duration.ofMillis(stopWatch.getTime()); + + if (duration.getSeconds() <= 10) { + return String.format("%dms", duration.toMillis()); + } else { + return String.format("%dm%ds", duration.toMinutes(), (duration.getSeconds() - (duration.toMinutes() * 60))); + } + } + + @Override + public void beforeCreate(GyroUI ui, State state, Resource resource) throws Exception { + start(resource); + } + + @Override + public void afterCreate(GyroUI ui, State state, Resource resource) throws Exception { + String elapsed = stop(ui, resource); + ui.write(ELAPSED_TIME_MESSAGE, elapsed); + } + + @Override + public void beforeUpdate( + GyroUI ui, State state, Resource current, Resource pending, Set changedFields) throws Exception { + start(pending); + } + + @Override + public void afterUpdate( + GyroUI ui, State state, Resource current, Resource pending, Set changedFields) throws Exception { + String elapsed = stop(ui, current); + ui.write(ELAPSED_TIME_MESSAGE, elapsed); + } + + @Override + public void beforeDelete(GyroUI ui, State state, Resource resource) throws Exception { + start(resource); + } + + @Override + public void afterDelete(GyroUI ui, State state, Resource resource) throws Exception { + String elapsed = stop(ui, resource); + ui.write(ELAPSED_TIME_MESSAGE, elapsed); + } +} diff --git a/core/src/main/java/gyro/core/scope/RootScope.java b/core/src/main/java/gyro/core/scope/RootScope.java index bd5a86806..37509040f 100644 --- a/core/src/main/java/gyro/core/scope/RootScope.java +++ b/core/src/main/java/gyro/core/scope/RootScope.java @@ -79,6 +79,7 @@ import gyro.core.resource.DiffableField; import gyro.core.resource.DiffableInternals; import gyro.core.resource.DiffableType; +import gyro.core.resource.ElapsedTimeChangeProcessor; import gyro.core.resource.ExtendsDirectiveProcessor; import gyro.core.resource.ModificationChangeProcessor; import gyro.core.resource.ModificationPlugin; @@ -178,6 +179,7 @@ public RootScope( Stream.of( new ModificationChangeProcessor(), + new ElapsedTimeChangeProcessor(), new ConfiguredFieldsChangeProcessor()) .forEach(p -> getSettings(ChangeSettings.class).getProcessors().add(p)); From 9e3980e58b24e17d73b0abb77ed2da27c0fd119d Mon Sep 17 00:00:00 2001 From: Deepanjan Bhattacharyya Date: Tue, 6 Sep 2022 13:37:57 -0700 Subject: [PATCH 12/12] preserve old ordering --- core/src/main/java/gyro/core/scope/RootScope.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/gyro/core/scope/RootScope.java b/core/src/main/java/gyro/core/scope/RootScope.java index 37509040f..0838b318c 100644 --- a/core/src/main/java/gyro/core/scope/RootScope.java +++ b/core/src/main/java/gyro/core/scope/RootScope.java @@ -179,8 +179,8 @@ public RootScope( Stream.of( new ModificationChangeProcessor(), - new ElapsedTimeChangeProcessor(), - new ConfiguredFieldsChangeProcessor()) + new ConfiguredFieldsChangeProcessor(), + new ElapsedTimeChangeProcessor()) .forEach(p -> getSettings(ChangeSettings.class).getProcessors().add(p)); Stream.of(