From 8e83f72e0c244f752f8f1daa4bed2a48fe6acbca Mon Sep 17 00:00:00 2001 From: Jon Schneider Date: Wed, 1 Mar 2017 17:09:57 -0800 Subject: [PATCH] Add dependency wayback machine --- build.gradle | 9 +- .../DependencyLockExtension.groovy | 4 + .../DependencyLockPlugin.groovy | 113 +++----- .../DependencyLockReader.groovy | 87 ++++++ .../DependencyLockWriter.groovy | 64 +++++ .../model/GradleDependency.groovy | 22 ++ .../{tasks => model}/LockKey.groovy | 6 +- .../dependencylock/model/LockValue.groovy | 14 + .../tasks/GenerateLockTask.groovy | 269 +++++++++--------- ...bstractNebulaMetricsWaybackProvider.groovy | 100 +++++++ .../NebulaMetricsCommitWaybackProvider.groovy | 22 ++ .../wayback/WaybackProvider.groovy | 29 ++ .../wayback/WaybackProviderFactory.groovy | 30 ++ .../nebula.metrics-commit.properties | 1 + .../DependencyLockLauncherSpec.groovy | 2 +- .../DependencyLockPluginSpec.groovy | 45 +++ .../DependencyLockReaderSpec.groovy | 6 + .../GenerateLockTaskWithWaybackSpec.groovy | 102 +++++++ ...ulaMetricsCommitWaybackProviderSpec.groovy | 106 +++++++ .../wayback/WaybackProviderFactorySpec.groovy | 29 ++ 20 files changed, 834 insertions(+), 226 deletions(-) create mode 100644 src/main/groovy/nebula/plugin/dependencylock/DependencyLockReader.groovy create mode 100644 src/main/groovy/nebula/plugin/dependencylock/DependencyLockWriter.groovy create mode 100644 src/main/groovy/nebula/plugin/dependencylock/model/GradleDependency.groovy rename src/main/groovy/nebula/plugin/dependencylock/{tasks => model}/LockKey.groovy (88%) create mode 100644 src/main/groovy/nebula/plugin/dependencylock/model/LockValue.groovy create mode 100644 src/main/groovy/nebula/plugin/dependencylock/wayback/AbstractNebulaMetricsWaybackProvider.groovy create mode 100644 src/main/groovy/nebula/plugin/dependencylock/wayback/NebulaMetricsCommitWaybackProvider.groovy create mode 100644 src/main/groovy/nebula/plugin/dependencylock/wayback/WaybackProvider.groovy create mode 100644 src/main/groovy/nebula/plugin/dependencylock/wayback/WaybackProviderFactory.groovy create mode 100644 src/main/resources/META-INF/dependency-wayback-provider/nebula.metrics-commit.properties create mode 100644 src/test/groovy/nebula/plugin/dependencylock/DependencyLockReaderSpec.groovy create mode 100644 src/test/groovy/nebula/plugin/dependencylock/tasks/GenerateLockTaskWithWaybackSpec.groovy create mode 100644 src/test/groovy/nebula/plugin/dependencylock/wayback/NebulaMetricsCommitWaybackProviderSpec.groovy create mode 100644 src/test/groovy/nebula/plugin/dependencylock/wayback/WaybackProviderFactorySpec.groovy diff --git a/build.gradle b/build.gradle index d08863f7..5a849998 100644 --- a/build.gradle +++ b/build.gradle @@ -19,6 +19,7 @@ plugins { id 'com.gradle.plugin-publish' version '0.9.7' id 'nebula.plugin-plugin' version '5.1.0' id 'org.ysb33r.gradletest' version '0.5.4' + id 'nebula.optional-base' version '3.0.3' } description 'Gradle plugin to allow locking of dynamic dependency versions' @@ -38,9 +39,15 @@ contacts { dependencies { compile 'com.netflix.nebula:gradle-scm-plugin:latest.release' + compile 'com.netflix.nebula:gradle-metrics-plugin:5.+', optional + compile 'org.codehaus.groovy.modules.http-builder:http-builder:0.7.1' testCompile 'com.netflix.nebula:gradle-git-scm-plugin:latest.release' testCompile 'org.ajoberstar:grgit:latest.release' + + // resolution rules requires at least 2.8.1, and currently DependencyLockLauncherSpec resolution-rules integration + // test will fail without this + testCompile 'com.fasterxml.jackson.core:jackson-databind:2.8.+' } pluginBundle { @@ -64,7 +71,7 @@ pluginBundle { } gradleTest { - versions '2.2.1', '2.3', '2.4', '2.5', '2.6', '2.7', '2.8', '2.9', '2.10', '2.11', '2.12', '2.13', '2.14.1', '3.0', '3.1' + versions '2.2.1', '2.3', '2.4', '2.5', '2.6', '2.7', '2.8', '2.9', '2.10', '2.11', '2.12', '2.13', '2.14.1', '3.0', '3.1', '3.2', '3.3' } gradleTest.dependsOn(jar) diff --git a/src/main/groovy/nebula/plugin/dependencylock/DependencyLockExtension.groovy b/src/main/groovy/nebula/plugin/dependencylock/DependencyLockExtension.groovy index 670032a9..04e74e8b 100644 --- a/src/main/groovy/nebula/plugin/dependencylock/DependencyLockExtension.groovy +++ b/src/main/groovy/nebula/plugin/dependencylock/DependencyLockExtension.groovy @@ -15,6 +15,9 @@ */ package nebula.plugin.dependencylock +import nebula.plugin.dependencylock.wayback.WaybackProvider +import nebula.plugin.dependencylock.wayback.WaybackProviderFactory + class DependencyLockExtension { String lockFile = 'dependencies.lock' String globalLockFile = 'global.lock' @@ -24,4 +27,5 @@ class DependencyLockExtension { Set skippedDependencies = [] as Set boolean includeTransitives = false boolean lockAfterEvaluating = true + Object waybackProvider } diff --git a/src/main/groovy/nebula/plugin/dependencylock/DependencyLockPlugin.groovy b/src/main/groovy/nebula/plugin/dependencylock/DependencyLockPlugin.groovy index 3c5702cf..6006c948 100644 --- a/src/main/groovy/nebula/plugin/dependencylock/DependencyLockPlugin.groovy +++ b/src/main/groovy/nebula/plugin/dependencylock/DependencyLockPlugin.groovy @@ -15,11 +15,12 @@ */ package nebula.plugin.dependencylock -import groovy.json.JsonSlurper import nebula.plugin.dependencylock.tasks.CommitLockTask import nebula.plugin.dependencylock.tasks.GenerateLockTask import nebula.plugin.dependencylock.tasks.SaveLockTask import nebula.plugin.dependencylock.tasks.UpdateLockTask +import nebula.plugin.dependencylock.wayback.WaybackProvider +import nebula.plugin.dependencylock.wayback.WaybackProviderFactory import nebula.plugin.scm.ScmPlugin import org.gradle.api.GradleException import org.gradle.api.Plugin @@ -32,7 +33,7 @@ import org.gradle.api.logging.Logging import org.gradle.api.tasks.Delete import org.gradle.util.NameMatcher -import static nebula.plugin.dependencylock.tasks.GenerateLockTask.getConfigurationsFromConfigurationNames +import static nebula.plugin.dependencylock.tasks.GenerateLockTask.lockableConfigurations class DependencyLockPlugin implements Plugin { private static final Logger LOGGER = Logging.getLogger(DependencyLockPlugin) @@ -45,24 +46,26 @@ class DependencyLockPlugin implements Plugin { private static final String USE_GENERATED_LOCK = 'dependencyLock.useGeneratedLock' private static final String USE_GENERATED_GLOBAL_LOCK = 'dependencyLock.useGeneratedGlobalLock' private static final String UPDATE_DEPENDENCIES = 'dependencyLock.updateDependencies' - private static final String OVERRIDE_FILE = 'dependencyLock.overrideFile' private static final String OVERRIDE = 'dependencyLock.override' - private static final List GENERATION_TASK_NAMES = [GENERATE_LOCK_TASK_NAME, GENERATE_GLOBAL_LOCK_TASK_NAME, UPDATE_LOCK_TASK_NAME, UPDATE_GLOBAL_LOCK_TASK_NAME] private static final List UPDATE_TASK_NAMES = [UPDATE_LOCK_TASK_NAME, UPDATE_GLOBAL_LOCK_TASK_NAME] + public static final String OVERRIDE_FILE = 'dependencyLock.overrideFile' public static final String GLOBAL_LOCK_CONFIG = '_global_' public static final String GENERATE_GLOBAL_LOCK_TASK_NAME = 'generateGlobalLock' public static final String UPDATE_GLOBAL_LOCK_TASK_NAME = 'updateGlobalLock' public static final String GENERATE_LOCK_TASK_NAME = 'generateLock' public static final String UPDATE_LOCK_TASK_NAME = 'updateLock' + public static final String WAYBACK_TASK_NAME = 'waybackLock' Project project + DependencyLockReader lockReader @Override void apply(Project project) { this.project = project + this.lockReader = new DependencyLockReader(project) DependencyLockExtension extension = project.extensions.create(EXTENSION_NAME, DependencyLockExtension) DependencyLockCommitExtension commitExtension = project.rootProject.extensions.findByType(DependencyLockCommitExtension) @@ -70,7 +73,7 @@ class DependencyLockPlugin implements Plugin { commitExtension = project.rootProject.extensions.create(COMMIT_EXTENSION_NAME, DependencyLockCommitExtension) } - Map overrides = loadOverrides() + Map overrides = lockReader.readOverrides() String globalLockFilename = project.hasProperty(GLOBAL_LOCK_FILE) ? project[GLOBAL_LOCK_FILE] : null String lockFilename = configureTasks(globalLockFilename, extension, commitExtension, overrides) @@ -115,13 +118,13 @@ class DependencyLockPlugin implements Plugin { String lockFilename = project.hasProperty(LOCK_FILE) ? project[LOCK_FILE] : null GenerateLockTask genLockTask = project.tasks.create(GENERATE_LOCK_TASK_NAME, GenerateLockTask) - configureLockTask(genLockTask, lockFilename, extension, overrides) + configureGenerateLockTask(genLockTask, lockFilename, extension, overrides) if (project.hasProperty(USE_GENERATED_LOCK)) { lockFilename = genLockTask.getDependenciesLock().path } UpdateLockTask updateLockTask = project.tasks.create(UPDATE_LOCK_TASK_NAME, UpdateLockTask) - configureLockTask(updateLockTask, lockFilename, extension, overrides) + configureGenerateLockTask(updateLockTask, lockFilename, extension, overrides) SaveLockTask saveTask = configureSaveTask(lockFilename, genLockTask, updateLockTask, extension) createDeleteLock(saveTask) @@ -245,9 +248,22 @@ class DependencyLockPlugin implements Plugin { globalSaveTask } - private GenerateLockTask configureLockTask(GenerateLockTask lockTask, String clLockFileName, DependencyLockExtension extension, Map overrides) { + private GenerateLockTask configureGenerateLockTask(GenerateLockTask lockTask, String clLockFileName, DependencyLockExtension extension, Map overrides) { setupLockConventionMapping(lockTask, extension, overrides) lockTask.conventionMapping.with { + waybackProvider = { + def impl = null + switch (extension.waybackProvider) { + case WaybackProvider: + impl = extension.waybackProvider + break + case String: + impl = new WaybackProviderFactory(project, getClass().classLoader).build(extension.waybackProvider as String) + break + } + impl + } + dependenciesLock = { new File(project.buildDir, clLockFileName ?: extension.lockFile) } @@ -281,7 +297,7 @@ class DependencyLockPlugin implements Plugin { def subprojects = project.subprojects.collect { subproject -> def ext = subproject.getExtensions().findByType(DependencyLockExtension) if (ext != null) { - def configurations = getConfigurationsFromConfigurationNames(project, subproject, ext.configurationNames) + def configurations = lockableConfigurations(project, subproject, ext.configurationNames) configurations.collect { configuration -> project.dependencies.create(project.dependencies.project(path: subproject.path, configuration: configuration.name)) } @@ -293,7 +309,7 @@ class DependencyLockPlugin implements Plugin { def conf = project.configurations.detachedConfiguration(subprojectsArray) project.allprojects.each { it.configurations.add(conf) } - [conf] + getConfigurationsFromConfigurationNames(project, project, extension.configurationNames) + [conf] + lockableConfigurations(project, project, extension.configurationNames) } } @@ -372,37 +388,15 @@ class DependencyLockPlugin implements Plugin { void applyLock(Configuration conf, File dependenciesLock, Collection updates = []) { LOGGER.info("Using ${dependenciesLock.name} to lock dependencies in $conf") - def locks = loadLock(dependenciesLock) - - if (updates) { - locks = locks.collectEntries { configurationName, deps -> - [(configurationName): deps.findAll { coord, info -> - def notUpdate = !updates.contains(coord) - def notTransitive = info.transitive == null - def hasRequestedVersion = info.requested != null - notUpdate && notTransitive && hasRequestedVersion - }] - } - } - - // in the old format, all first level props were groupId:artifactId - def isDeprecatedFormat = !locks.isEmpty() && locks.every { it.key ==~ /[^:]+:.+/ } - // in the old format, all first level props were groupId:artifactId - if (isDeprecatedFormat) { - LOGGER.warn("${dependenciesLock.name} is using a deprecated lock format. Support for this format may be removed in future versions.") + def locks = lockReader.readLocks(conf, dependenciesLock, updates) + + if(locks) { + // Non-project locks are the top-level dependencies, and possibly transitive thereof, of this project which are + // locked by the lock file. There may also be dependencies on other projects. These are not captured here. + def locked = locks.findAll { it.value?.locked }.collect { "${it.key}:${it.value.locked}" } + LOGGER.debug('locked: {}', locked) + lockConfiguration(conf, locked) } - - // In the old format of the lock file, there was only one locked setting. In that case, apply it on all configurations. - // In the new format, apply _global_ to all configurations or use the config name - def notations = isDeprecatedFormat ? locks : locks[GLOBAL_LOCK_CONFIG] ?: locks[conf.name] - - // Non-project locks are the top-level dependencies, and possibly transitive thereof, of this project which are - // locked by the lock file. There may also be dependencies on other projects. These are not captured here. - def locked = notations.findAll { it.value?.locked }.collect { "${it.key}:${it.value.locked}" } - - LOGGER.debug('locked: {}', locked) - - lockConfiguration(conf, locked) } void applyOverrides(Configuration conf, Map overrides) { @@ -437,43 +431,4 @@ class DependencyLockPlugin implements Plugin { false } } - - private Map loadOverrides() { - // Overrides are dependencies that trump the lock file. - Map overrides = [:] - - // Load overrides from a file if the user has specified one via a property. - if (project.hasProperty(OVERRIDE_FILE)) { - File dependenciesLock = new File(project.rootDir, project[OVERRIDE_FILE] as String) - def lockOverride = loadLock(dependenciesLock) - def isDeprecatedFormat = lockOverride.any { it.value.getClass() != String && it.value.locked } - // the old lock override files specified the version to override under the "locked" property - if (isDeprecatedFormat) { - LOGGER.warn("The override file ${dependenciesLock.name} is using a deprecated format. Support for this format may be removed in future versions.") - } - lockOverride.each { overrides[it.key] = isDeprecatedFormat ? it.value.locked : it.value } - LOGGER.debug "Override file loaded: ${project[OVERRIDE_FILE]}" - } - - // Allow the user to specify overrides via a property as well. - if (project.hasProperty('dependencyLock.override')) { - project['dependencyLock.override'].tokenize(',').each { - def (group, artifact, version) = it.tokenize(':') - overrides["${group}:${artifact}".toString()] = version - LOGGER.debug "Override added for: ${it}" - } - } - - return overrides - } - - private static loadLock(File lock) { - try { - return new JsonSlurper().parseText(lock.text) - } catch (ex) { - LOGGER.debug('Unreadable json file: ' + lock.text) - LOGGER.error('JSON unreadable') - throw new GradleException("${lock.name} is unreadable or invalid json, terminating run", ex) - } - } } diff --git a/src/main/groovy/nebula/plugin/dependencylock/DependencyLockReader.groovy b/src/main/groovy/nebula/plugin/dependencylock/DependencyLockReader.groovy new file mode 100644 index 00000000..5f9b3bbf --- /dev/null +++ b/src/main/groovy/nebula/plugin/dependencylock/DependencyLockReader.groovy @@ -0,0 +1,87 @@ +package nebula.plugin.dependencylock + +import groovy.json.JsonSlurper +import groovy.transform.TupleConstructor +import org.gradle.api.GradleException +import org.gradle.api.Project +import org.gradle.api.artifacts.Configuration +import org.gradle.api.logging.Logger +import org.gradle.api.logging.Logging +import static nebula.plugin.dependencylock.DependencyLockPlugin.OVERRIDE_FILE +import static nebula.plugin.dependencylock.DependencyLockPlugin.GLOBAL_LOCK_CONFIG + +@TupleConstructor +class DependencyLockReader { + private static final Logger logger = Logging.getLogger(DependencyLockPlugin) + + Project project + + Map readLocks(Configuration conf, File dependenciesLock, Collection updates = []) { + logger.info("Using ${dependenciesLock.name} to lock dependencies in $conf") + if(!dependenciesLock.exists()) + return null + + Map locks = parseLockFile(dependenciesLock) + + if (updates) { + locks = locks.collectEntries { configurationName, deps -> + [(configurationName): deps.findAll { coord, info -> + def notUpdate = !updates.contains(coord) + def notTransitive = info.transitive == null + def hasRequestedVersion = info.requested != null + notUpdate && notTransitive && hasRequestedVersion + }] + } + } + + // in the old format, all first level props were groupId:artifactId + def isDeprecatedFormat = !locks.isEmpty() && locks.every { it.key ==~ /[^:]+:.+/ } + // in the old format, all first level props were groupId:artifactId + if (isDeprecatedFormat) { + logger.warn("${dependenciesLock.name} is using a deprecated lock format. Support for this format may be removed in future versions.") + } + + // In the old format of the lock file, there was only one locked setting. In that case, apply it on all configurations. + // In the new format, apply _global_ to all configurations or use the config name + return isDeprecatedFormat ? locks : locks[GLOBAL_LOCK_CONFIG] ?: locks[conf.name] + } + + Map readOverrides() { + // Overrides are dependencies that trump the lock file. + Map overrides = [:] + + // Load overrides from a file if the user has specified one via a property. + if (project.hasProperty(OVERRIDE_FILE)) { + File dependenciesLock = new File(project.rootDir, project[OVERRIDE_FILE] as String) + def lockOverride = parseLockFile(dependenciesLock) + def isDeprecatedFormat = lockOverride.any { it.value.getClass() != String && it.value.locked } + // the old lock override files specified the version to override under the "locked" property + if (isDeprecatedFormat) { + logger.warn("The override file ${dependenciesLock.name} is using a deprecated format. Support for this format may be removed in future versions.") + } + lockOverride.each { overrides[it.key] = isDeprecatedFormat ? it.value.locked : it.value } + logger.debug "Override file loaded: ${project[OVERRIDE_FILE]}" + } + + // Allow the user to specify overrides via a property as well. + if (project.hasProperty('dependencyLock.override')) { + project['dependencyLock.override'].tokenize(',').each { + def (group, artifact, version) = it.tokenize(':') + overrides["${group}:${artifact}".toString()] = version + logger.debug "Override added for: ${it}" + } + } + + return overrides + } + + private static Map parseLockFile(File lock) { + try { + return new JsonSlurper().parseText(lock.text) as Map + } catch (ex) { + logger.debug('Unreadable json file: ' + lock.text) + logger.error('JSON unreadable') + throw new GradleException("${lock.name} is unreadable or invalid json, terminating run", ex) + } + } +} diff --git a/src/main/groovy/nebula/plugin/dependencylock/DependencyLockWriter.groovy b/src/main/groovy/nebula/plugin/dependencylock/DependencyLockWriter.groovy new file mode 100644 index 00000000..fa12b1ac --- /dev/null +++ b/src/main/groovy/nebula/plugin/dependencylock/DependencyLockWriter.groovy @@ -0,0 +1,64 @@ +package nebula.plugin.dependencylock + +import groovy.json.JsonOutput +import groovy.transform.Canonical +import nebula.plugin.dependencylock.model.LockKey +import nebula.plugin.dependencylock.model.LockValue + +import static nebula.plugin.dependencylock.DependencyLockPlugin.GLOBAL_LOCK_CONFIG + +@Canonical +class DependencyLockWriter { + File lockFile + Set skippedDependencies + + void writeLock(Map locks) { + lockFile.parentFile.mkdirs() + + // The result map maps configuration -> map of group:artifact -> map of dep properties -> values. The result + // would then be transformed into Json. For example: + // { + // "runtime": { + // "test.example:foo": { locked: "2.0.0", "transitive: ["test:sub1", "test:sub2"] } + // "test:sub1": { "project: true" } + // "test:sub2": { "project: true" } + // } + // "default": { + // "test.example:foo": { locked: "2.0.0", "transitive: ["test:sub1", "test:sub2"] } + // "test:sub1": { "project: true" } + // "test:sub2": { "project: true" } + // } + // } + def result = new TreeMap().withDefault { new TreeMap().withDefault { new TreeMap() } } + + def filteredSkippedDeps = locks.findAll { LockKey k, v -> + !skippedDependencies.contains("${k.group}:${k.artifact}" as String) + } + + filteredSkippedDeps.each { key, lock -> + def configuration = key.configuration.startsWith('detachedConfiguration') ? GLOBAL_LOCK_CONFIG : key.configuration + def depMap = result[configuration]["${key.group}:${key.artifact}"] + if (lock.locked) { + depMap['locked'] = lock.locked + } else { + depMap['project'] = true + } + if (lock.requested) { + depMap['requested'] = lock.requested + } + if (lock.viaOverride) { + depMap['viaOverride'] = lock.viaOverride + } + if (lock.transitive) { + def transitiveFrom = lock.transitive.collect { "${it.group}:${it.artifact}" }.sort() + depMap['transitive'] = transitiveFrom + } + if (lock.firstLevelTransitive) { + def transitiveFrom = lock.firstLevelTransitive.collect { "${it.group}:${it.artifact}" }.sort() + depMap['firstLevelTransitive'] = transitiveFrom + } + } + + lockFile.text = JsonOutput.prettyPrint(JsonOutput.toJson(result)) + } +} diff --git a/src/main/groovy/nebula/plugin/dependencylock/model/GradleDependency.groovy b/src/main/groovy/nebula/plugin/dependencylock/model/GradleDependency.groovy new file mode 100644 index 00000000..83b62805 --- /dev/null +++ b/src/main/groovy/nebula/plugin/dependencylock/model/GradleDependency.groovy @@ -0,0 +1,22 @@ +package nebula.plugin.dependencylock.model + +import groovy.transform.Canonical + +@Canonical +class GradleDependency { + String group + String name + String version + + static GradleDependency fromConstant(Object expr) { + def matcher = expr =~ /(?[^:]+)?(:(?[^:]+))(:(?[^@:]+))/ + if (matcher.matches()) { + return new GradleDependency( + matcher.group('group'), + matcher.group('name'), + matcher.group('version') + ) + } + return null + } +} \ No newline at end of file diff --git a/src/main/groovy/nebula/plugin/dependencylock/tasks/LockKey.groovy b/src/main/groovy/nebula/plugin/dependencylock/model/LockKey.groovy similarity index 88% rename from src/main/groovy/nebula/plugin/dependencylock/tasks/LockKey.groovy rename to src/main/groovy/nebula/plugin/dependencylock/model/LockKey.groovy index 1723c341..49dd489f 100644 --- a/src/main/groovy/nebula/plugin/dependencylock/tasks/LockKey.groovy +++ b/src/main/groovy/nebula/plugin/dependencylock/model/LockKey.groovy @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nebula.plugin.dependencylock.tasks +package nebula.plugin.dependencylock.model -import groovy.transform.EqualsAndHashCode +import groovy.transform.Canonical /** * Map key for locked dependencies. */ -@EqualsAndHashCode +@Canonical class LockKey { String group String artifact diff --git a/src/main/groovy/nebula/plugin/dependencylock/model/LockValue.groovy b/src/main/groovy/nebula/plugin/dependencylock/model/LockValue.groovy new file mode 100644 index 00000000..1dae45ce --- /dev/null +++ b/src/main/groovy/nebula/plugin/dependencylock/model/LockValue.groovy @@ -0,0 +1,14 @@ +package nebula.plugin.dependencylock.model + +import groovy.transform.Canonical + +@Canonical +class LockValue { + Set transitive = [] as Set + Set firstLevelTransitive = [] as Set + Boolean childrenVisited = false + String requested + String locked + Boolean project = false + String viaOverride +} \ No newline at end of file diff --git a/src/main/groovy/nebula/plugin/dependencylock/tasks/GenerateLockTask.groovy b/src/main/groovy/nebula/plugin/dependencylock/tasks/GenerateLockTask.groovy index 127dd274..1e272720 100644 --- a/src/main/groovy/nebula/plugin/dependencylock/tasks/GenerateLockTask.groovy +++ b/src/main/groovy/nebula/plugin/dependencylock/tasks/GenerateLockTask.groovy @@ -15,7 +15,12 @@ */ package nebula.plugin.dependencylock.tasks -import groovy.json.JsonOutput +import nebula.plugin.dependencylock.DependencyLockReader +import nebula.plugin.dependencylock.DependencyLockWriter +import nebula.plugin.dependencylock.exceptions.DependencyLockException +import nebula.plugin.dependencylock.model.LockKey +import nebula.plugin.dependencylock.model.LockValue +import nebula.plugin.dependencylock.wayback.WaybackProvider import org.gradle.api.Project import org.gradle.api.artifacts.Configuration import org.gradle.api.artifacts.Dependency @@ -23,8 +28,6 @@ import org.gradle.api.artifacts.ExternalDependency import org.gradle.api.artifacts.ResolvedDependency import org.gradle.api.tasks.TaskAction -import static nebula.plugin.dependencylock.DependencyLockPlugin.GLOBAL_LOCK_CONFIG - class GenerateLockTask extends AbstractLockTask { String description = 'Create a lock file in build/' Collection configurations = [] @@ -32,17 +35,20 @@ class GenerateLockTask extends AbstractLockTask { Closure filter = { group, name, version -> true } Set skippedDependencies = [] File dependenciesLock - Map overrides + Map overrides boolean includeTransitives = false + WaybackProvider waybackProvider @TaskAction void lock() { - Collection confs = getConfigurations() ?: getConfigurationsFromConfigurationNames(project, project, getConfigurationNames()) - def dependencyMap = readDependenciesFromConfigurations(confs) - writeLock(dependencyMap) + Collection confs = getConfigurations() ?: lockableConfigurations(project, project, getConfigurationNames()) + Map dependencyMap = project.hasProperty('waybackTo') ? + new GenerateLockFromWayback().lock(confs) : + new GenerateLockFromConfigurations().lock(confs) + new DependencyLockWriter(getDependenciesLock(), getSkippedDependencies()).writeLock(dependencyMap) } - public static Collection getConfigurationsFromConfigurationNames(Project taskProject, Project project, Set configurationNames) { + static Collection lockableConfigurations(Project taskProject, Project project, Set configurationNames) { if (configurationNames.empty) { if (Configuration.class.declaredMethods.any { it.name == 'isCanBeResolved' }) { project.configurations.findAll { @@ -60,172 +66,151 @@ class GenerateLockTask extends AbstractLockTask { } } - Map readDependenciesFromConfigurations(Collection confs) { - def deps = [:].withDefault { [transitive: [] as Set, firstLevelTransitive: [] as Set, childrenVisited: false] } - - // Peers are all the projects in the build to which this plugin has been applied. - def peers = project.rootProject.allprojects.collect { new LockKey(group: it.group, artifact: it.name) } + class GenerateLockFromConfigurations { + Map lock(Collection confs) { + Map deps = [:].withDefault { new LockValue() } - confs.each { Configuration configuration -> - // Capture the version of each dependency as requested in the build script for reference. - def externalDependencies = configuration.allDependencies.withType(ExternalDependency) - def filteredExternalDependencies = externalDependencies.findAll { Dependency dependency -> - filter(dependency.group, dependency.name, dependency.version) - } - filteredExternalDependencies.each { ExternalDependency dependency -> - def key = new LockKey(group: dependency.group, artifact: dependency.name, configuration: configuration.name) - deps[key].requested = dependency.version - } + // Peers are all the projects in the build to which this plugin has been applied. + def peers = project.rootProject.allprojects.collect { new LockKey(group: it.group, artifact: it.name) } - // Lock the version of each dependency specified in the build script as resolved by Gradle. - def resolvedDependencies = configuration.resolvedConfiguration.firstLevelModuleDependencies - def filteredResolvedDependencies = resolvedDependencies.findAll { ResolvedDependency resolved -> - filter(resolved.moduleGroup, resolved.moduleName, resolved.moduleVersion) - } + confs.each { Configuration configuration -> + // Capture the version of each dependency as requested in the build script for reference. + def externalDependencies = configuration.allDependencies.withType(ExternalDependency) + def filteredExternalDependencies = externalDependencies.findAll { Dependency dependency -> + filter(dependency.group, dependency.name, dependency.version) + } + filteredExternalDependencies.each { ExternalDependency dependency -> + def key = new LockKey(group: dependency.group, artifact: dependency.name, configuration: configuration.name) + deps[key].requested = dependency.version + } - filteredResolvedDependencies.each { ResolvedDependency resolved -> - def key = new LockKey(group: resolved.moduleGroup, artifact: resolved.moduleName, configuration: configuration.name) + // Lock the version of each dependency specified in the build script as resolved by Gradle. + def resolvedDependencies = configuration.resolvedConfiguration.firstLevelModuleDependencies + def filteredResolvedDependencies = resolvedDependencies.findAll { ResolvedDependency resolved -> + filter(resolved.moduleGroup, resolved.moduleName, resolved.moduleVersion) + } - // If this dependency does not exist in our list of peers, it is a standard dependency. Otherwise, it is - // a project dependency. - if (!isKeyInPeerList(key, peers)) { - deps[key].locked = resolved.moduleVersion - } else { - // Project dependencies don't have a version so they must be treated differently. Record the project - // as an explicit dependency, but do not lock it to a version. - deps[key].project = true + filteredResolvedDependencies.each { ResolvedDependency resolved -> + def key = new LockKey(group: resolved.moduleGroup, artifact: resolved.moduleName, configuration: configuration.name) - // If we don't include transitive dependencies, then we must lock the first-level "transitive" - // dependencies of each project dependency. - if (!getIncludeTransitives()) { - handleSiblingTransitives(resolved, configuration.name, deps, peers) + // If this dependency does not exist in our list of peers, it is a standard dependency. Otherwise, it is + // a project dependency. + if (!isKeyInPeerList(key, peers)) { + deps[key].locked = resolved.moduleVersion + } else { + // Project dependencies don't have a version so they must be treated differently. Record the project + // as an explicit dependency, but do not lock it to a version. + deps[key].project = true + + // If we don't include transitive dependencies, then we must lock the first-level "transitive" + // dependencies of each project dependency. + if (!getIncludeTransitives()) { + handleSiblingTransitives(resolved, configuration.name, deps, peers) + } } - } - // If requested, lock all the transitive dependencies of the declared top-level dependencies. - if (getIncludeTransitives()) { - deps[key].childrenVisited = true - resolved.children.each { handleTransitive(it, configuration.name, deps, peers, key) } + // If requested, lock all the transitive dependencies of the declared top-level dependencies. + if (getIncludeTransitives()) { + deps[key].childrenVisited = true + resolved.children.each { handleTransitive(it, configuration.name, deps, peers, key) } + } } } - } - // Add all the overrides to the locked dependencies and record whether a specified override modified a - // preexisting dependency. - getOverrides().each { String k, String overrideVersion -> - def (overrideGroup, overrideArtifact) = k.tokenize(':') - deps.each { depLockKey, depValue -> - if (depLockKey.group == overrideGroup && depLockKey.artifact == overrideArtifact) { - depValue.viaOverride = overrideVersion + // Add all the overrides to the locked dependencies and record whether a specified override modified a + // preexisting dependency. + getOverrides().each { String k, String overrideVersion -> + def (overrideGroup, overrideArtifact) = k.tokenize(':') + deps.each { depLockKey, depValue -> + if (depLockKey.group == overrideGroup && depLockKey.artifact == overrideArtifact) { + depValue.viaOverride = overrideVersion + } } } - } - return deps - } + return deps + } - void handleSiblingTransitives(ResolvedDependency sibling, String configName, Map deps, List peers) { - def parent = new LockKey(group: sibling.moduleGroup, artifact: sibling.moduleName, configuration: sibling.configuration) - sibling.children.each { ResolvedDependency dependency -> - def key = new LockKey(group: dependency.moduleGroup, artifact: dependency.moduleName, configuration: configName) + private void handleSiblingTransitives(ResolvedDependency sibling, String configName, Map deps, List peers) { + def parent = new LockKey(group: sibling.moduleGroup, artifact: sibling.moduleName, configuration: sibling.configuration) + sibling.children.each { ResolvedDependency dependency -> + def key = new LockKey(group: dependency.moduleGroup, artifact: dependency.moduleName, configuration: configName) - // Record the project[s] from which this dependency originated. - deps[key].firstLevelTransitive << parent + // Record the project[s] from which this dependency originated. + deps[key].firstLevelTransitive << parent - // Lock the transitive dependencies of each project dependency, recursively. - if (isKeyInPeerList(key, peers)) { - deps[key].project = true + // Lock the transitive dependencies of each project dependency, recursively. + if (isKeyInPeerList(key, peers)) { + deps[key].project = true - // Multiple configurations may specify dependencies on the same project, and multiple projects might - // also be dependent on the same project. We only need to record the top-level transitive dependencies - // once for each project. Flag a project as visited as soon as we encounter it. - if ((dependency.children.size() > 0) && !deps[key].childrenVisited) { - deps[key].childrenVisited = true - handleSiblingTransitives(dependency, configName, deps, peers) + // Multiple configurations may specify dependencies on the same project, and multiple projects might + // also be dependent on the same project. We only need to record the top-level transitive dependencies + // once for each project. Flag a project as visited as soon as we encounter it. + if ((dependency.children.size() > 0) && !deps[key].childrenVisited) { + deps[key].childrenVisited = true + handleSiblingTransitives(dependency, configName, deps, peers) + } + } else { + deps[key].locked = dependency.moduleVersion } - } else { - deps[key].locked = dependency.moduleVersion } } - } - void handleTransitive(ResolvedDependency transitive, String configName, Map deps, List peers, LockKey parent) { - def key = new LockKey(group: transitive.moduleGroup, artifact: transitive.moduleName, configuration: configName) + private void handleTransitive(ResolvedDependency transitive, String configName, Map deps, List peers, LockKey parent) { + def key = new LockKey(group: transitive.moduleGroup, artifact: transitive.moduleName, configuration: configName) - // Multiple dependencies may share any subset of their transitive dependencies. Each dependency only needs to be - // visited once so flag it once we visit it. - if (!deps[key].childrenVisited) { + // Multiple dependencies may share any subset of their transitive dependencies. Each dependency only needs to be + // visited once so flag it once we visit it. + if (!deps[key].childrenVisited) { - // Lock each dependency and its children, recursively. Don't forget transitive project dependencies. - if (!isKeyInPeerList(key, peers)) { - deps[key].locked = transitive.moduleVersion - } else { - deps[key].project = true - } - if (transitive.children.size() > 0) { - deps[key].childrenVisited = true + // Lock each dependency and its children, recursively. Don't forget transitive project dependencies. + if (!isKeyInPeerList(key, peers)) { + deps[key].locked = transitive.moduleVersion + } else { + deps[key].project = true + } + if (transitive.children.size() > 0) { + deps[key].childrenVisited = true + } + transitive.children.each { handleTransitive(it, configName, deps, peers, key) } } - transitive.children.each { handleTransitive(it, configName, deps, peers, key) } - } - - // Record the dependencies from which this artifact originated transitively. - deps[key].transitive << parent - } - def isKeyInPeerList(LockKey lockKey, List peers) { - return peers.any { - it.group == lockKey.group && it.artifact == lockKey.artifact + // Record the dependencies from which this artifact originated transitively. + deps[key].transitive << parent } - } - void writeLock(deps) { - - // The result map maps configuration -> map of group:artifact -> map of dep properties -> values. The result - // would then be transformed into Json. For example: - // { - // "runtime": { - // "test.example:foo": { locked: "2.0.0", "transitive: ["test:sub1", "test:sub2"] } - // "test:sub1": { "project: true" } - // "test:sub2": { "project: true" } - // } - // "default": { - // "test.example:foo": { locked: "2.0.0", "transitive: ["test:sub1", "test:sub2"] } - // "test:sub1": { "project: true" } - // "test:sub2": { "project: true" } - // } - // } - - def result = new TreeMap().withDefault { new TreeMap().withDefault { new TreeMap() } } - - def filteredSkippedDeps = deps.findAll { - LockKey k, v -> !getSkippedDependencies().contains("${k.group}:${k.artifact}" as String) + private static def isKeyInPeerList(LockKey lockKey, List peers) { + return peers.any { + it.group == lockKey.group && it.artifact == lockKey.artifact + } } + } - filteredSkippedDeps.each { key, lock -> - def configuration = key.configuration.startsWith('detachedConfiguration') ? GLOBAL_LOCK_CONFIG : key.configuration - def depMap = result[configuration]["${key.group}:${key.artifact}"] - if (lock.locked) { - depMap['locked'] = lock.locked - } else { - depMap['project'] = true - } - if (lock.requested) { - depMap['requested'] = lock.requested - } - if (lock.viaOverride) { - depMap['viaOverride'] = lock.viaOverride + class GenerateLockFromWayback { + Map lock(Collection confs) { + if(!waybackProvider) { + throw new DependencyLockException("In order to use wayback, you must configure a provider") } - if (lock.transitive) { - def transitiveFrom = lock.transitive.collect { "${it.group}:${it.artifact}" }.sort() - depMap['transitive'] = transitiveFrom + + DependencyLockReader reader = new DependencyLockReader(project) + + def updateLocks = confs.collectEntries { conf -> + [(conf.name): reader.readLocks(conf, getDependenciesLock(), [])?.collectEntries { k, v -> + // parse the string representation returned by readLocks into (LockKey, LockValue) entries + def (group, artifact) = (k as String).split(':') + [(new LockKey(group, artifact, conf.name)): v as LockValue] + } ] } - if (lock.firstLevelTransitive) { - def transitiveFrom = lock.firstLevelTransitive.collect { "${it.group}:${it.artifact}" }.sort() - depMap['firstLevelTransitive'] = transitiveFrom + + confs.each { conf -> + def wayback = waybackProvider.wayback(project.property('waybackTo') as String, conf) + // we only override or create a new configuration to lock if wayback has some advice about it + if(!wayback.isEmpty()) + updateLocks[conf.name] = wayback.collectEntries { dep -> [(new LockKey(dep.group, dep.name, conf.name)): new LockValue(locked: dep.version)] } } - } - project.buildDir.mkdirs() - getDependenciesLock().text = JsonOutput.prettyPrint(JsonOutput.toJson(result)) + return updateLocks.values().findAll { it }.sum() as Map + } } } + diff --git a/src/main/groovy/nebula/plugin/dependencylock/wayback/AbstractNebulaMetricsWaybackProvider.groovy b/src/main/groovy/nebula/plugin/dependencylock/wayback/AbstractNebulaMetricsWaybackProvider.groovy new file mode 100644 index 00000000..8d719d70 --- /dev/null +++ b/src/main/groovy/nebula/plugin/dependencylock/wayback/AbstractNebulaMetricsWaybackProvider.groovy @@ -0,0 +1,100 @@ +package nebula.plugin.dependencylock.wayback + +import com.google.gson.FieldNamingPolicy +import com.google.gson.GsonBuilder +import groovy.transform.Memoized +import groovyx.net.http.HttpResponseException +import groovyx.net.http.RESTClient +import io.searchbox.client.JestClient +import io.searchbox.client.JestClientFactory +import io.searchbox.client.config.HttpClientConfig +import io.searchbox.client.http.JestHttpClient +import io.searchbox.core.Search +import io.searchbox.core.SearchResult +import nebula.plugin.dependencylock.model.GradleDependency +import nebula.plugin.metrics.MetricsPluginExtension +import net.logstash.logback.encoder.org.apache.commons.lang.StringUtils +import org.gradle.api.GradleException +import org.gradle.api.Project +import org.joda.time.DateTime + +abstract class AbstractNebulaMetricsWaybackProvider extends WaybackProvider { + RESTClient client + + AbstractNebulaMetricsWaybackProvider(Project project) { + super(project) + def metrics = project.extensions.findByType(MetricsPluginExtension) + + if(!metrics) { + throw new GradleException('You must apply the nebula.metrics plugin in order to use this wayback provider') + } + + client = new RESTClient("http://$metrics.hostname:$metrics.httpPort") + } + + /** + * @param terms + * @return Map of configuration names to their set of resolved dependencies + */ + @Memoized + protected Map> waybackAllConfigurations(Map terms, Closure maxHit) { + String query = """ + |{ + | "query": { + | "filtered": { + | "query": { + | "match_all": {} + | }, + | "filter": { + | "and": [ + """.stripMargin() + + terms.collect { k, v -> """ + | { + | "term": { + | "$k": "$v" + | } + | } + """.stripMargin()}.join(',') + """ + | ] + | } + | } + | } + |} + """.stripMargin() + + def hit = null + + for(int i = 0; i <= 6; i++) { + try { + def resp = client.post(path: "/${getIndexName(project, DateTime.now().minusMonths(i))}/build/_search", + body: query, + requestContentType: 'application/json') + + if(resp.status == 200) { + hit = resp.data.hits?.hits?.collect { it._source }?.max(maxHit) + if (hit) break + } + } catch (HttpResponseException ignored) { + // Jest throws an exception attempting to parse JSON bodies when a web server returns an HTML body with a 404, + // so just do nothing here... (bug in Abstract.createNewElasticSearchResult) + } + } + + /* + "resolved-dependencies": { + "myapp-dependencies": {}, + "myapp-app-dependencies": { + "Resolved-Dependencies-ResolutionRules": "com.netflix.nebula:gradle-resolution-rules:0.41.0,netflix.nebula.resolutionrules:resolution-rules:0.33.0", + ... + */ + Map projectDependencies = hit?.'resolved-dependencies'?.find { it.key.startsWith(project.name) }?.value + return ((projectDependencies?.collectEntries { k, v -> + [(StringUtils.uncapitalize(StringUtils.substringAfter(k, 'Resolved-Dependencies-'))) : + v.split(',').collect { GradleDependency.fromConstant(it) }.toSet()] + } ?: [:]) as Map>).withDefault {[] as Set} + } + + protected static String getIndexName(Project project, DateTime dt) { + return project.extensions.findByType(MetricsPluginExtension).getIndexName(dt) + } +} diff --git a/src/main/groovy/nebula/plugin/dependencylock/wayback/NebulaMetricsCommitWaybackProvider.groovy b/src/main/groovy/nebula/plugin/dependencylock/wayback/NebulaMetricsCommitWaybackProvider.groovy new file mode 100644 index 00000000..7d851bc8 --- /dev/null +++ b/src/main/groovy/nebula/plugin/dependencylock/wayback/NebulaMetricsCommitWaybackProvider.groovy @@ -0,0 +1,22 @@ +package nebula.plugin.dependencylock.wayback + +import nebula.plugin.dependencylock.model.GradleDependency +import org.gradle.api.Project +import org.gradle.api.artifacts.Configuration +import org.joda.time.DateTime + +class NebulaMetricsCommitWaybackProvider extends AbstractNebulaMetricsWaybackProvider { + NebulaMetricsCommitWaybackProvider(Project project) { + super(project) + } + + @Override + Set wayback(String selector, Configuration configuration) { + Map> depsByConf = waybackAllConfigurations( + ['project.name': project.rootProject.name, 'info.scm.generic.change': selector], + { DateTime.parse(it.startTime as String) } + ) + + return depsByConf[configuration.name] + } +} \ No newline at end of file diff --git a/src/main/groovy/nebula/plugin/dependencylock/wayback/WaybackProvider.groovy b/src/main/groovy/nebula/plugin/dependencylock/wayback/WaybackProvider.groovy new file mode 100644 index 00000000..9ae24761 --- /dev/null +++ b/src/main/groovy/nebula/plugin/dependencylock/wayback/WaybackProvider.groovy @@ -0,0 +1,29 @@ +package nebula.plugin.dependencylock.wayback + +import groovy.transform.TupleConstructor +import nebula.plugin.dependencylock.model.GradleDependency +import org.gradle.api.Incubating +import org.gradle.api.Project +import org.gradle.api.artifacts.Configuration + +/** + * Provides information about dependencies used at some point in the past so we can "rewind" our dependency + * lock back to that time. Provider implementations can be provided for a variety of different backing datastores (e.g. + * Nebula Metrics) and selectors (e.g. past commit, last tagged release, dependencies used in the context of a particular + * deployed application). + */ +@Incubating +abstract class WaybackProvider { + Project project + + WaybackProvider(Project project) { this.project = project } + + /** + * @param selector - The point in time at which to lock (could be a timestamp, a version number, + * or any other ordered sequence supported by a provider implementation). + * @param configuration - The configuration to retrieve dependencies for. Some providers may only be able to + * provide runtime dependencies, etc. depending on what their backing datasources are. + * @return The dependencies in configuration as of selector. + */ + abstract Set wayback(String selector, Configuration configuration) +} \ No newline at end of file diff --git a/src/main/groovy/nebula/plugin/dependencylock/wayback/WaybackProviderFactory.groovy b/src/main/groovy/nebula/plugin/dependencylock/wayback/WaybackProviderFactory.groovy new file mode 100644 index 00000000..1785beba --- /dev/null +++ b/src/main/groovy/nebula/plugin/dependencylock/wayback/WaybackProviderFactory.groovy @@ -0,0 +1,30 @@ +package nebula.plugin.dependencylock.wayback + +import groovy.transform.TupleConstructor +import org.gradle.api.GradleException +import org.gradle.api.Project +import org.gradle.util.GUtil + +@TupleConstructor +class WaybackProviderFactory { + Project project + ClassLoader classLoader + + private String findProviderClassName(String providerId) { + URL propertiesFileUrl = classLoader.getResource(String.format("META-INF/dependency-wayback-provider/%s.properties", providerId)) + return propertiesFileUrl ? GUtil.loadProperties(propertiesFileUrl).getProperty('implementation-class') : null + } + + WaybackProvider build(String providerId) { + def providerClassName = findProviderClassName(providerId) + if (!providerClassName) + throw new GradleException(String.format("No implementation class or includes specified for provider '%s' in %s.", providerId, providerClassName)) + + try { + return classLoader.loadClass(providerClassName).newInstance(project) as WaybackProvider + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) { + throw new GradleException(String.format("Could not find or load implementation class '%s' for rule '%s'.", + providerClassName, providerId), e) + } + } +} diff --git a/src/main/resources/META-INF/dependency-wayback-provider/nebula.metrics-commit.properties b/src/main/resources/META-INF/dependency-wayback-provider/nebula.metrics-commit.properties new file mode 100644 index 00000000..2d45c161 --- /dev/null +++ b/src/main/resources/META-INF/dependency-wayback-provider/nebula.metrics-commit.properties @@ -0,0 +1 @@ +implementation-class: nebula.plugin.dependencylock.wayback.NebulaMetricsCommitWaybackProvider \ No newline at end of file diff --git a/src/test/groovy/nebula/plugin/dependencylock/DependencyLockLauncherSpec.groovy b/src/test/groovy/nebula/plugin/dependencylock/DependencyLockLauncherSpec.groovy index 4fab397b..55e80894 100644 --- a/src/test/groovy/nebula/plugin/dependencylock/DependencyLockLauncherSpec.groovy +++ b/src/test/groovy/nebula/plugin/dependencylock/DependencyLockLauncherSpec.groovy @@ -1059,7 +1059,7 @@ class DependencyLockLauncherSpec extends IntegrationSpec { def 'generateLock interacts well with resolution rules'() { buildFile << """\ plugins { - id "nebula.resolution-rules" version "2.0.0" + id "nebula.resolution-rules" version "2.4.1" } apply plugin: 'nebula.dependency-lock' diff --git a/src/test/groovy/nebula/plugin/dependencylock/DependencyLockPluginSpec.groovy b/src/test/groovy/nebula/plugin/dependencylock/DependencyLockPluginSpec.groovy index b67254f2..a7cd328e 100644 --- a/src/test/groovy/nebula/plugin/dependencylock/DependencyLockPluginSpec.groovy +++ b/src/test/groovy/nebula/plugin/dependencylock/DependencyLockPluginSpec.groovy @@ -16,8 +16,12 @@ package nebula.plugin.dependencylock import nebula.plugin.dependencylock.dependencyfixture.Fixture +import nebula.plugin.dependencylock.model.GradleDependency +import nebula.plugin.dependencylock.wayback.NebulaMetricsCommitWaybackProvider +import nebula.plugin.dependencylock.wayback.WaybackProvider import nebula.test.ProjectSpec import org.gradle.api.Project +import org.gradle.api.artifacts.Configuration import org.gradle.testfixtures.ProjectBuilder class DependencyLockPluginSpec extends ProjectSpec { @@ -420,6 +424,47 @@ class DependencyLockPluginSpec extends ProjectSpec { foo2.moduleVersion.id.version == '1.0.1' } + def 'configure wayback provider by id'() { + when: + project.with { + buildscript { + repositories { jcenter() } + dependencies { classpath 'com.netflix.nebula:gradle-metrics-plugin:5.+' } + } + + apply plugin: 'nebula.metrics' + apply plugin: pluginName + dependencyLock.waybackProvider = 'nebula.metrics-commit' + } + + then: + project.tasks.generateLock.waybackProvider instanceof NebulaMetricsCommitWaybackProvider + } + + def 'configure wayback provider with an inline custom implementation'() { + when: + project.with { + buildscript { + repositories { jcenter() } + dependencies { classpath 'com.netflix.nebula:gradle-metrics-plugin:5.+' } + } + + apply plugin: 'nebula.metrics' + apply plugin: pluginName + dependencyLock.waybackProvider = new WaybackProvider(project) { + String name = 'custom' + + @Override + Set wayback(String selector, Configuration configuration) { + return [] as Set + } + } + } + + then: + project.tasks.generateLock.waybackProvider.name == 'custom' + } + private List multiProjectSetup() { def sub1Folder = new File(projectDir, 'sub1') sub1Folder.mkdir() diff --git a/src/test/groovy/nebula/plugin/dependencylock/DependencyLockReaderSpec.groovy b/src/test/groovy/nebula/plugin/dependencylock/DependencyLockReaderSpec.groovy new file mode 100644 index 00000000..11d5ccd1 --- /dev/null +++ b/src/test/groovy/nebula/plugin/dependencylock/DependencyLockReaderSpec.groovy @@ -0,0 +1,6 @@ +package nebula.plugin.dependencylock + +import spock.lang.Specification + +class DependencyLockReaderSpec extends Specification { +} diff --git a/src/test/groovy/nebula/plugin/dependencylock/tasks/GenerateLockTaskWithWaybackSpec.groovy b/src/test/groovy/nebula/plugin/dependencylock/tasks/GenerateLockTaskWithWaybackSpec.groovy new file mode 100644 index 00000000..de18018a --- /dev/null +++ b/src/test/groovy/nebula/plugin/dependencylock/tasks/GenerateLockTaskWithWaybackSpec.groovy @@ -0,0 +1,102 @@ +package nebula.plugin.dependencylock.tasks + +import groovy.json.JsonSlurper +import nebula.plugin.dependencylock.dependencyfixture.Fixture +import nebula.plugin.dependencylock.model.GradleDependency +import nebula.plugin.dependencylock.wayback.WaybackProvider +import nebula.test.ProjectSpec +import org.gradle.api.artifacts.Configuration + +class GenerateLockTaskWithWaybackSpec extends ProjectSpec { + final String taskName = 'generateLock' + GenerateLockTask task + + def setup() { + Fixture.createFixtureIfNotCreated() + + project.with { + apply plugin: 'java' + repositories { maven { url Fixture.repo } } + configurations { integTestCompile } + dependencies { + compile 'test.example:foo:2.+' + testCompile 'test.example:baz:1.+' + } + } + + task = project.tasks.create(taskName, GenerateLockTask) + task.configurationNames = [ 'compile', 'testCompile', 'integTestCompile' ] + task.waybackProvider = new WaybackProvider(project) { + @Override + Set wayback(String selector, Configuration configuration) { + if(configuration.name == 'compile') { + [new GradleDependency('test.example', 'foo', 'WAYBACK')] as Set + } + else if(configuration.name == 'testCompile') { + [new GradleDependency('test.example', 'baz', 'WAYBACK')] as Set + } + else [] + } + } + + task.dependenciesLock = new File(project.buildDir, 'dependencies.lock') + project.buildDir.mkdirs() + + // simulate user input via -PwaybackTo=previous with an extension property + project.ext.waybackTo = 'previous' + } + + def 'lock both configurations using wayback data'() { + when: + task.execute() + def locks = new JsonSlurper().parse(task.dependenciesLock) + + then: + locks.compile.'test.example:foo'.locked == 'WAYBACK' + locks.testCompile.'test.example:baz'.locked == 'WAYBACK' + } + + def 'update previously locked configurations using wayback data'() { + setup: + task.dependenciesLock.text = ''' + |{ + | "testCompile": { + | "test.example:baz": { + | "locked": "ORIGINAL", + | "requested": "1.+" + | } + | } + |} + '''.stripMargin() + + when: + task.execute() + def locks = new JsonSlurper().parse(task.dependenciesLock) + + then: + locks.testCompile.'test.example:baz'.locked == 'WAYBACK' + } + + def "don't update previously locked configurations that have no wayback input"() { + setup: + task.dependenciesLock.text = ''' + |{ + | "integTestCompile": { + | "test.example:baz": { + | "locked": "ORIGINAL", + | "requested": "1.+" + | } + | } + |} + '''.stripMargin() + + when: + task.execute() + def locks = new JsonSlurper().parse(task.dependenciesLock) + + then: + // this is a configuration for which our notional wayback provider does not have any information, + // leave existing locks in place + locks.integTestCompile.'test.example:baz'.locked == 'ORIGINAL' + } +} diff --git a/src/test/groovy/nebula/plugin/dependencylock/wayback/NebulaMetricsCommitWaybackProviderSpec.groovy b/src/test/groovy/nebula/plugin/dependencylock/wayback/NebulaMetricsCommitWaybackProviderSpec.groovy new file mode 100644 index 00000000..b6e8ccea --- /dev/null +++ b/src/test/groovy/nebula/plugin/dependencylock/wayback/NebulaMetricsCommitWaybackProviderSpec.groovy @@ -0,0 +1,106 @@ +package nebula.plugin.dependencylock.wayback + +import com.sun.net.httpserver.HttpExchange +import com.sun.net.httpserver.HttpHandler +import com.sun.net.httpserver.HttpServer +import nebula.plugin.metrics.MetricsPluginExtension +import org.gradle.api.Project +import org.gradle.testfixtures.ProjectBuilder +import org.joda.time.DateTime +import spock.lang.Specification + +class NebulaMetricsCommitWaybackProviderSpec extends Specification { + def rootProjectName = 'myapp' + def projectName = 'myapp-app' + + Project p + MetricsPluginExtension metricsExt + NebulaMetricsCommitWaybackProvider provider + HttpServer esServer + + def setup() { + p = ProjectBuilder.builder() + .withParent(ProjectBuilder.builder().withName(rootProjectName).build()) + .withName(projectName).build() + + metricsExt = p.extensions.create('metrics', MetricsPluginExtension).with { + hostname = 'localhost' + httpPort = 7104 + rollingIndex = true + delegate + } + + provider = new NebulaMetricsCommitWaybackProvider(p) + + esServer = mockServer() + } + + def cleanup() { + esServer.stop(0) + } + + def 'extract resolved dependencies from a Nebula Metrics response'() { + when: + handleAndReturnHits(esServer, metricsExt.getIndexName(), 'compile', 'com.google.guava:guava:19.0') + esServer.start() + then: + provider.wayback('mycommit', p.configurations.create('compile')).size() == 1 + } + + def 'return no results when the metrics server is unavailable'() { + when: + esServer.start() // everything will return 404 + + then: + provider.wayback('mycommit', p.configurations.create('compile')).isEmpty() + } + + def "when the most recent build is in a prior month's metrics index, find it still"() { + when: + handleAndReturnHits(esServer, metricsExt.getIndexName(DateTime.now().minusMonths(1)), 'compile', 'com.google.guava:guava:19.0') + esServer.start() + + then: + provider.wayback('mycommit', p.configurations.create('compile')).size() == 1 + } + + private static HttpServer mockServer() { + return HttpServer.create(new InetSocketAddress(7104), 0) + } + + private void handleAndReturnHits(HttpServer server, String index, String configuration, String... dependencies) { + server.createContext("/$index/build/_search", new HttpHandler() { + @Override + void handle(HttpExchange t) throws IOException { + println(t.requestURI) + def response = """ + |{ + | "hits": { + | "hits": [ + | { + | "_index": "$index", + | "_type": "build", + | "_source": { + | "startTime": "2016-08-30T19:21:34.447Z", + | "resolved-dependencies": { + | "$rootProjectName-dependencies": {}, + | "$projectName-dependencies": { + | "Resolved-Dependencies-${configuration.capitalize()}": "${dependencies.join(',')}" + | } + | } + | } + | } + | ] + | } + |} + """.stripMargin() + + t.responseHeaders.add('Content-Type', 'application/json') + t.sendResponseHeaders(200, response.length()) + OutputStream os = t.getResponseBody() + os.write(response.getBytes()) + os.close() + } + }) + } +} diff --git a/src/test/groovy/nebula/plugin/dependencylock/wayback/WaybackProviderFactorySpec.groovy b/src/test/groovy/nebula/plugin/dependencylock/wayback/WaybackProviderFactorySpec.groovy new file mode 100644 index 00000000..92de5ff6 --- /dev/null +++ b/src/test/groovy/nebula/plugin/dependencylock/wayback/WaybackProviderFactorySpec.groovy @@ -0,0 +1,29 @@ +package nebula.plugin.dependencylock.wayback + +import nebula.plugin.metrics.MetricsPluginExtension +import org.gradle.api.GradleException +import org.gradle.api.Project +import org.gradle.testfixtures.ProjectBuilder +import spock.lang.Specification + +class WaybackProviderFactorySpec extends Specification { + Project project = ProjectBuilder.builder().build() + WaybackProviderFactory factory = new WaybackProviderFactory(project, getClass().classLoader) + + def setup() { + project.extensions.create('metrics', MetricsPluginExtension) + } + + def 'construct a wayback provider by provider ID'() { + expect: + factory.build('nebula.metrics-commit') + } + + def 'return null when the provider ID has no matching resource on the classpath'() { + when: + factory.build('nebula.does-not-exist') + + then: + thrown(GradleException) + } +}