Skip to content

Issue #78 Fixed thread safety for multiproject builds with --parallel #79

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cbue wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Issue #78 Fixed thread safety for multiproject builds with --parallel #79

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
76f5324
Issue #78 Fixed thread safety for multiproject builds with --parallel
Dec 19, 2017
2ef2639
Issue #78 Fixed thread-safety for DependencyLockProvider, IvyRecommen…
Dec 19, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 20 additions & 12 deletions src/main/groovy/netflix/nebula/dependency/recommender/provider/DependencyLockProvider.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import groovy.json.JsonSlurper
import org.gradle.api.Project

class DependencyLockProvider extends FileBasedRecommendationProvider {
Map<String, String> recommendations
volatile Map<String, String> recommendations

DependencyLockProvider() {}

Expand All @@ -14,20 +14,28 @@ class DependencyLockProvider extends FileBasedRecommendationProvider {

@Override
String getVersion(String org, String name) throws Exception {
if (!recommendations) {
input.withCloseable {
final locks = new JsonSlurper().parse(it)
final isDependencyLock4Format = locks.every {
it.value.every {
it.value instanceof Map

Map<String, String> tmpResult = recommendations

if (tmpResult == null) {
synchronized (this) {
tmpResult = recommendations
if (tmpResult == null) {
input.withCloseable {
final locks = new JsonSlurper().parse(it)
final isDependencyLock4Format = locks.every {
it.value.every {
it.value instanceof Map
}
}
tmpResult = (isDependencyLock4Format ? locks.collectEntries { it.value } : locks).collectEntries {
[(it.key): it.value.locked]
}
}
}

recommendations = (isDependencyLock4Format ? locks.collectEntries { it.value } : locks).collectEntries {
[(it.key): it.value.locked]
recommendations = tmpResult
}
}
}
recommendations[org + ':' + name]
tmpResult[org + ':' + name]
}
}
26 changes: 17 additions & 9 deletions ...in/groovy/netflix/nebula/dependency/recommender/provider/IvyRecommendationProvider.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,33 @@ package netflix.nebula.dependency.recommender.provider
import org.gradle.api.Project

class IvyRecommendationProvider extends FileBasedRecommendationProvider {
Map<String, String> versionsByCoord
volatile Map<String, String> versionsByCoord

IvyRecommendationProvider(Project p) { super(p) }

@Override
String getVersion(String org, String name) throws Exception {
if (versionsByCoord == null) {
versionsByCoord = [:]
getInput().withCloseable {
def ivy = new XmlSlurper().parse(it)
ivy.dependencies.dependency.each { d ->
versionsByCoord.put("${[email protected]()}:${[email protected]()}".toString(), "${[email protected]()}")

Map<String, String> tmpResult = versionsByCoord

if (tmpResult == null) {
synchronized (this) {
tmpResult = versionsByCoord
if (tmpResult == null) {
tmpResult = [:]
getInput().withCloseable {
def ivy = new XmlSlurper().parse(it)
ivy.dependencies.dependency.each { d ->
tmpResult.put("${[email protected]()}:${[email protected]()}".toString(), "${[email protected]()}")
}
}
versionsByCoord = tmpResult
}
}
}
return versionsByCoord["$org:$name".toString()]
return tmpResult["$org:$name".toString()]
}

@SuppressWarnings("unchecked")
@Override
public InputStreamProvider setModule(Object dependencyNotation) {
if (dependencyNotation == null)
Expand Down
183 changes: 99 additions & 84 deletions ...groovy/netflix/nebula/dependency/recommender/provider/MavenBomRecommendationProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,41 +15,44 @@
*/
package netflix.nebula.dependency.recommender.provider;

import com.netflix.nebula.dependencybase.DependencyManagement;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.maven.model.Dependency;
import org.apache.maven.model.Model;
import org.apache.maven.model.Parent;
import org.apache.maven.model.Repository;
import org.apache.maven.model.building.*;
import org.apache.maven.model.building.DefaultModelBuilder;
import org.apache.maven.model.building.DefaultModelBuilderFactory;
import org.apache.maven.model.building.DefaultModelBuildingRequest;
import org.apache.maven.model.building.ModelBuildingRequest;
import org.apache.maven.model.building.ModelBuildingResult;
import org.apache.maven.model.building.ModelProblemCollector;
import org.apache.maven.model.building.ModelSource2;
import org.apache.maven.model.interpolation.StringSearchModelInterpolator;
import org.apache.maven.model.path.DefaultPathTranslator;
import org.apache.maven.model.path.DefaultUrlNormalizer;
import org.apache.maven.model.resolution.InvalidRepositoryException;
import org.apache.maven.model.resolution.ModelResolver;
import org.apache.maven.model.resolution.UnresolvableModelException;
import org.codehaus.groovy.runtime.EncodingGroovyMethods;
import org.codehaus.plexus.interpolation.MapBasedValueSource;
import org.codehaus.plexus.interpolation.PropertiesBasedValueSource;
import org.codehaus.plexus.interpolation.ValueSource;
import org.gradle.api.Project;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.ModuleVersionIdentifier;
import org.gradle.api.artifacts.ResolvedConfiguration;
import org.gradle.api.artifacts.repositories.ArtifactRepository;
import org.gradle.api.artifacts.repositories.MavenArtifactRepository;

import java.io.*;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.net.URLConnection;
import java.util.*;
import org.gradle.api.artifacts.repositories.PasswordCredentials;
import org.gradle.api.internal.artifacts.DefaultModuleVersionIdentifier;
import org.gradle.internal.impldep.com.google.common.base.Throwables;
import com.netflix.nebula.dependencybase.DependencyManagement;

public class MavenBomRecommendationProvider extends ClasspathBasedRecommendationProvider {
private Map<String, String> recommendations;
private volatile Map<String, String> recommendations;
private DependencyManagement insight;

public MavenBomRecommendationProvider(Project project, String configName) {
Expand Down Expand Up @@ -96,78 +99,90 @@ public String getVersion(String org, String name) throws Exception {
}

public Map<String, String> getRecommendations() throws Exception {
if(recommendations == null) {
recommendations = new HashMap<>();

Set<File> recommendationFiles = getFilesOnConfiguration();
for (File recommendation : recommendationFiles) {
if (!recommendation.getName().endsWith("pom")) {
break;
}

DefaultModelBuildingRequest request = new DefaultModelBuildingRequest();

request.setModelResolver(new ModelResolver() {
@Override
public ModelSource2 resolveModel(String groupId, String artifactId, String version) throws UnresolvableModelException {
String notation = groupId + ":" + artifactId + ":" + version + "@pom";
org.gradle.api.artifacts.Dependency dependency = project.getDependencies().create(notation);
Configuration configuration = project.getConfigurations().detachedConfiguration(dependency);
try {
File file = configuration.getFiles().iterator().next();
return new SimpleModelSource(new FileInputStream(file));
} catch (Exception e) {
throw new UnresolvableModelException(e, groupId, artifactId, version);

Map<String, String> tmpResult = recommendations;

if(tmpResult == null) {

synchronized (this) {

tmpResult = recommendations;
if (tmpResult == null) {

tmpResult = new HashMap<>();

Set<File> recommendationFiles = getFilesOnConfiguration();
for (File recommendation : recommendationFiles) {
if (!recommendation.getName().endsWith("pom")) {
break;
}

DefaultModelBuildingRequest request = new DefaultModelBuildingRequest();

request.setModelResolver(new ModelResolver() {
@Override
public ModelSource2 resolveModel(String groupId, String artifactId, String version) throws UnresolvableModelException {
String notation = groupId + ":" + artifactId + ":" + version + "@pom";
org.gradle.api.artifacts.Dependency dependency = project.getDependencies().create(notation);
Configuration configuration = project.getConfigurations().detachedConfiguration(dependency);
try {
File file = configuration.getFiles().iterator().next();
return new SimpleModelSource(new FileInputStream(file));
} catch (Exception e) {
throw new UnresolvableModelException(e, groupId, artifactId, version);
}
}

@Override
public ModelSource2 resolveModel(Dependency dependency) throws UnresolvableModelException {
return resolveModel(dependency.getGroupId(), dependency.getArtifactId(), dependency.getVersion());
}

@Override
public ModelSource2 resolveModel(Parent parent) throws UnresolvableModelException {
return resolveModel(parent.getGroupId(), parent.getArtifactId(), parent.getVersion());
}

@Override
public void addRepository(Repository repository) throws InvalidRepositoryException {
// do nothing
}

@Override
public void addRepository(Repository repository, boolean bool) throws InvalidRepositoryException {
// do nothing
}

@Override
public ModelResolver newCopy() {
return this; // do nothing
}
});
request.setModelSource(new SimpleModelSource(new FileInputStream(recommendation)));
request.setSystemProperties(System.getProperties());

DefaultModelBuilder modelBuilder = new DefaultModelBuilderFactory().newInstance();
modelBuilder.setModelInterpolator(new ProjectPropertiesModelInterpolator(project));

ModelBuildingResult result = modelBuilder.build(request);
insight.addPluginMessage("nebula.dependency-recommender uses mavenBom: " + result.getEffectiveModel().getId());
Model model = result.getEffectiveModel();
if (model == null) {
break;
}
org.apache.maven.model.DependencyManagement dependencyManagement = model.getDependencyManagement();
if (dependencyManagement == null) {
break;
}
for (Dependency d : dependencyManagement.getDependencies()) {
tmpResult.put(d.getGroupId() + ":" + d.getArtifactId(), d.getVersion());
}
}

@Override
public ModelSource2 resolveModel(Dependency dependency) throws UnresolvableModelException {
return resolveModel(dependency.getGroupId(), dependency.getArtifactId(), dependency.getVersion());
}

@Override
public ModelSource2 resolveModel(Parent parent) throws UnresolvableModelException {
return resolveModel(parent.getGroupId(), parent.getArtifactId(), parent.getVersion());
}

@Override
public void addRepository(Repository repository) throws InvalidRepositoryException {
// do nothing
}

@Override
public void addRepository(Repository repository, boolean bool) throws InvalidRepositoryException {
// do nothing
}

@Override
public ModelResolver newCopy() {
return this; // do nothing
}
});
request.setModelSource(new SimpleModelSource(new FileInputStream(recommendation)));
request.setSystemProperties(System.getProperties());

DefaultModelBuilder modelBuilder = new DefaultModelBuilderFactory().newInstance();
modelBuilder.setModelInterpolator(new ProjectPropertiesModelInterpolator(project));

ModelBuildingResult result = modelBuilder.build(request);
insight.addPluginMessage("nebula.dependency-recommender uses mavenBom: " + result.getEffectiveModel().getId());
Model model = result.getEffectiveModel();
if (model == null) {
break;
}
org.apache.maven.model.DependencyManagement dependencyManagement = model.getDependencyManagement();
if (dependencyManagement == null) {
break;
}
for (Dependency d : dependencyManagement.getDependencies()) {
recommendations.put(d.getGroupId() + ":" + d.getArtifactId(), d.getVersion());
recommendations = tmpResult;
}
}
}
return recommendations;
return tmpResult;
}

private static class ProjectPropertiesModelInterpolator extends StringSearchModelInterpolator {
Expand Down
23 changes: 16 additions & 7 deletions ...vy/netflix/nebula/dependency/recommender/provider/PropertyFileRecommendationProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
import java.util.Properties;

public class PropertyFileRecommendationProvider extends FileBasedRecommendationProvider {
private Properties recommendations;

private volatile Properties recommendations;

private FuzzyVersionResolver fuzzyResolver = new FuzzyVersionResolver() {
@Override
Expand All @@ -31,12 +32,20 @@ public PropertyFileRecommendationProvider(Project project) {

@Override
public String getVersion(String org, String name) throws Exception {
if(recommendations == null) {
recommendations = new Properties();
try (InputStream inputStream = inputProvider.getInputStream()) {
recommendations.load(
new EolCommentFilteringReader(
new ColonFilteringReader(new InputStreamReader(inputStream))));

Properties tmpResult = recommendations;

if(tmpResult == null) {
synchronized (this) {
if (tmpResult == null) {
tmpResult = new Properties();
try (InputStream inputStream = inputProvider.getInputStream()) {
tmpResult.load(
new EolCommentFilteringReader(
new ColonFilteringReader(new InputStreamReader(inputStream))));
}
recommendations = tmpResult;
}
}
}
return fuzzyResolver.versionOf(org + "/" + name);
Expand Down