Skip to content

Commit

Permalink
Add supporting S3 tags
Browse files Browse the repository at this point in the history
  • Loading branch information
Anatoliy Korostelev committed Nov 13, 2024
1 parent 3d7ac57 commit 306ef00
Show file tree
Hide file tree
Showing 16 changed files with 180 additions and 20 deletions.
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
<properties>
<changelist>999999-SNAPSHOT</changelist>
<gitHubRepo>jenkinsci/s3-plugin</gitHubRepo>
<jenkins.version>2.387.3</jenkins.version>
<jenkins.version>2.414.3</jenkins.version>
</properties>

<developers>
Expand Down
8 changes: 7 additions & 1 deletion src/main/java/hudson/plugins/s3/Entry.java
Original file line number Diff line number Diff line change
Expand Up @@ -90,11 +90,16 @@ public final class Entry implements Describable<Entry> {
*/
public List<MetadataPair> userMetadata;

/**
* Tags overrides
*/
public List<TagPair> userTags;

@DataBoundConstructor
public Entry(String bucket, String sourceFile, String excludedFile, String storageClass, String selectedRegion,
boolean noUploadOnFailure, boolean uploadFromSlave, boolean managedArtifacts,
boolean useServerSideEncryption, boolean flatten, boolean gzipFiles, boolean keepForever,
boolean showDirectlyInBrowser, List<MetadataPair> userMetadata) {
boolean showDirectlyInBrowser, List<MetadataPair> userMetadata, List<TagPair> userTags) {
this.bucket = bucket;
this.sourceFile = sourceFile;
this.excludedFile = excludedFile;
Expand All @@ -108,6 +113,7 @@ public Entry(String bucket, String sourceFile, String excludedFile, String stora
this.gzipFiles = gzipFiles;
this.keepForever = keepForever;
this.userMetadata = userMetadata;
this.userTags = userTags;
this.showDirectlyInBrowser = showDirectlyInBrowser;
}

Expand Down
49 changes: 47 additions & 2 deletions src/main/java/hudson/plugins/s3/S3BucketPublisher.java
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,13 @@ public final class S3BucketPublisher extends Recorder implements SimpleBuildStep
*/
private /*almost final*/ List<MetadataPair> userMetadata;

/**
* User tag key/value pairs to tag the upload with.
*/
private /*almost final*/ List<TagPair> userTags;

@DataBoundConstructor
public S3BucketPublisher(String profileName, List<Entry> entries, List<MetadataPair> userMetadata,
public S3BucketPublisher(String profileName, List<Entry> entries, List<MetadataPair> userMetadata, List<TagPair> userTags,
boolean dontWaitForConcurrentBuildCompletion, String consoleLogLevel, String pluginFailureResultConstraint,
boolean dontSetBuildResultOnFailure) {
if (profileName == null) {
Expand All @@ -104,6 +109,11 @@ public S3BucketPublisher(String profileName, List<Entry> entries, List<MetadataP
this.userMetadata = new ArrayList<>();
}

this.userTags = userTags;
if (this.userTags == null) {

Check warning on line 113 in src/main/java/hudson/plugins/s3/S3BucketPublisher.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Partially covered line

Line 113 is only partially covered, one branch is missing
this.userTags = new ArrayList<>();
}

this.dontWaitForConcurrentBuildCompletion = dontWaitForConcurrentBuildCompletion;
this.dontSetBuildResultOnFailure = dontSetBuildResultOnFailure;
this.consoleLogLevel = parseLevel(consoleLogLevel);
Expand All @@ -129,6 +139,9 @@ protected Object readResolve() {
if (userMetadata == null)
userMetadata = new ArrayList<>();

if (userTags == null)
userTags = new ArrayList<>();

if (pluginFailureResultConstraint == null)
pluginFailureResultConstraint = Result.FAILURE;

Expand Down Expand Up @@ -172,6 +185,11 @@ public List<MetadataPair> getUserMetadata() {
return userMetadata;
}

@SuppressWarnings("unused")
public List<TagPair> getUserTags() {
return userTags;

Check warning on line 190 in src/main/java/hudson/plugins/s3/S3BucketPublisher.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered lines

Lines 114-190 are not covered by tests
}

@SuppressWarnings("unused")
public String getProfileName() {
return this.profileName;
Expand Down Expand Up @@ -317,9 +335,10 @@ public void perform(@NonNull Run<?, ?> run, @NonNull FilePath ws, @NonNull Launc


final Map<String, String> escapedMetadata = buildMetadata(envVars, entry);
final Map<String, String> escapedTags = buildTags(envVars, entry);

final List<FingerprintRecord> records = Lists.newArrayList();
final List<FingerprintRecord> fingerprints = profile.upload(run, bucket, paths, filenames, escapedMetadata, storageClass, selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption, entry.gzipFiles);
final List<FingerprintRecord> fingerprints = profile.upload(run, bucket, paths, filenames, escapedMetadata, escapedTags, storageClass, selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption, entry.gzipFiles);

for (FingerprintRecord fingerprintRecord : fingerprints) {
records.add(fingerprintRecord);
Expand Down Expand Up @@ -420,6 +439,32 @@ private Map<String, String> buildMetadata(Map<String, String> envVars, Entry ent
return escapedMetadata;
}

private Map<String, String> buildTags(Map<String, String> envVars, Entry entry) {
final Map<String, String> mergedTags = new HashMap<>();

if (userTags != null) {

Check warning on line 445 in src/main/java/hudson/plugins/s3/S3BucketPublisher.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Partially covered line

Line 445 is only partially covered, one branch is missing
for (TagPair pair : userTags) {

Check warning on line 446 in src/main/java/hudson/plugins/s3/S3BucketPublisher.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Partially covered line

Line 446 is only partially covered, one branch is missing
mergedTags.put(pair.key, pair.value);
}

Check warning on line 448 in src/main/java/hudson/plugins/s3/S3BucketPublisher.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered lines

Lines 447-448 are not covered by tests
}

if (entry.userTags != null) {
for (TagPair pair : entry.userTags) {

Check warning on line 452 in src/main/java/hudson/plugins/s3/S3BucketPublisher.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Partially covered line

Line 452 is only partially covered, one branch is missing
mergedTags.put(pair.key, pair.value);
}

Check warning on line 454 in src/main/java/hudson/plugins/s3/S3BucketPublisher.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered lines

Lines 453-454 are not covered by tests
}

final Map<String, String> escapedTags = new HashMap<>();

for (Map.Entry<String, String> mapEntry : mergedTags.entrySet()) {

Check warning on line 459 in src/main/java/hudson/plugins/s3/S3BucketPublisher.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Partially covered line

Line 459 is only partially covered, one branch is missing
escapedTags.put(
Util.replaceMacro(mapEntry.getKey(), envVars),
Util.replaceMacro(mapEntry.getValue(), envVars));
}

Check warning on line 463 in src/main/java/hudson/plugins/s3/S3BucketPublisher.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered lines

Lines 460-463 are not covered by tests

return escapedTags;
}

private String getFilename(FilePath src, boolean flatten, int searchIndex) {
final String fileName;
if (flatten) {
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/hudson/plugins/s3/S3Profile.java
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ public List<FingerprintRecord> upload(Run<?, ?> run,
final List<FilePath> filePaths,
final List<String> fileNames,
final Map<String, String> userMetadata,
final Map<String, String> userTags,
final String storageClass,
final String selregion,
final boolean uploadFromSlave,
Expand All @@ -151,10 +152,10 @@ public List<FingerprintRecord> upload(Run<?, ?> run,

final MasterSlaveCallable<String> upload;
if (gzipFiles) {
upload = new S3GzipCallable(accessKey, secretKey, useRole, dest, userMetadata,
upload = new S3GzipCallable(accessKey, secretKey, useRole, dest, userMetadata, userTags,

Check warning on line 155 in src/main/java/hudson/plugins/s3/S3Profile.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered line

Line 155 is not covered by tests
storageClass, selregion, useServerSideEncryption, getProxy());
} else {
upload = new S3UploadCallable(accessKey, secretKey, useRole, dest, userMetadata,
upload = new S3UploadCallable(accessKey, secretKey, useRole, dest, userMetadata, userTags,
storageClass, selregion, useServerSideEncryption, getProxy());
}

Expand Down
43 changes: 43 additions & 0 deletions src/main/java/hudson/plugins/s3/TagPair.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package hudson.plugins.s3;

import hudson.Extension;
import hudson.model.Describable;
import hudson.model.Descriptor;
import org.kohsuke.stapler.DataBoundConstructor;

public final class TagPair implements Describable<TagPair> {

/**
* The key of the user tag pair to tag an upload with.
* Can contain macros.
*/
public String key;

Check warning

Code scanning / Jenkins Security Scan

Jenkins: Plaintext password storage Warning

Field should be reviewed whether it stores a password and is serialized to disk: key

/**
* The key of the user tag pair to tag an upload with.
* Can contain macros.
*/
public String value;

@DataBoundConstructor
public TagPair(String key, String value) {
this.key = key;
this.value = value;
}

public Descriptor<TagPair> getDescriptor() {
return DESCRIPOR;

Check warning on line 29 in src/main/java/hudson/plugins/s3/TagPair.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered lines

Lines 23-29 are not covered by tests
}

@Extension
public final static DescriptorImpl DESCRIPOR = new DescriptorImpl();

public static class DescriptorImpl extends Descriptor<TagPair> {

@Override
public String getDisplayName() {
return "Tag";
}
};
}

5 changes: 3 additions & 2 deletions src/main/java/hudson/plugins/s3/Uploads.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.ObjectTagging;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
Expand All @@ -21,8 +22,8 @@ private Uploads() {}
private final transient HashMap<FilePath, Upload> startedUploads = new HashMap<>();
private final transient HashMap<FilePath, InputStream> openedStreams = new HashMap<>();

public Upload startUploading(TransferManager manager, FilePath file, InputStream inputsStream, String bucketName, String objectName, ObjectMetadata metadata) throws AmazonServiceException {
final PutObjectRequest request = new PutObjectRequest(bucketName, objectName, inputsStream, metadata);
public Upload startUploading(TransferManager manager, FilePath file, InputStream inputsStream, String bucketName, String objectName, ObjectMetadata metadata, ObjectTagging tags) throws AmazonServiceException {
final PutObjectRequest request = new PutObjectRequest(bucketName, objectName, inputsStream, metadata).withTagging(tags);

Check warning on line 26 in src/main/java/hudson/plugins/s3/Uploads.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered line

Line 26 is not covered by tests

// Set the buffer size (ReadLimit) equal to the multipart upload size,
// allowing us to resend data if the connection breaks.
Expand Down
21 changes: 18 additions & 3 deletions src/main/java/hudson/plugins/s3/callable/S3BaseUploadCallable.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import com.amazonaws.services.s3.internal.Mimetypes;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.ObjectTagging;
import com.amazonaws.services.s3.model.Tag;
import hudson.FilePath;
import hudson.ProxyConfiguration;
import hudson.plugins.s3.Destination;
Expand All @@ -12,24 +14,28 @@
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

public abstract class S3BaseUploadCallable extends S3Callable<String> {
private static final long serialVersionUID = 1L;
private final Destination dest;
private final String storageClass;
private final Map<String, String> userMetadata;
private final Map<String, String> userTags;
private final boolean useServerSideEncryption;


public S3BaseUploadCallable(String accessKey, Secret secretKey, boolean useRole,
Destination dest, Map<String, String> userMetadata, String storageClass, String selregion,
boolean useServerSideEncryption, ProxyConfiguration proxy) {
Destination dest, Map<String, String> userMetadata, Map<String, String> userTags,
String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy) {
super(accessKey, secretKey, useRole, selregion, proxy);
this.dest = dest;
this.storageClass = storageClass;
this.userMetadata = userMetadata;
this.userTags = userTags;
this.useServerSideEncryption = useServerSideEncryption;
}

Expand Down Expand Up @@ -86,6 +92,15 @@ protected ObjectMetadata buildMetadata(FilePath filePath) throws IOException, In
return metadata;
}

protected ObjectTagging s3Tagging() {

final List<Tag> tags = userTags.entrySet().stream()
.map(entry -> new Tag(entry.getKey(), entry.getValue()))
.collect(Collectors.toList());

return new ObjectTagging(tags);

Check warning on line 101 in src/main/java/hudson/plugins/s3/callable/S3BaseUploadCallable.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered lines

Lines 97-101 are not covered by tests
}

public Destination getDest() {
return dest;
}
Expand Down
8 changes: 5 additions & 3 deletions src/main/java/hudson/plugins/s3/callable/S3GzipCallable.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package hudson.plugins.s3.callable;

import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.ObjectTagging;
import com.amazonaws.services.s3.transfer.Upload;
import com.amazonaws.event.ProgressListener;
import com.amazonaws.event.ProgressEvent;
Expand All @@ -23,8 +24,8 @@
import java.util.zip.GZIPOutputStream;

public final class S3GzipCallable extends S3BaseUploadCallable implements MasterSlaveCallable<String> {
public S3GzipCallable(String accessKey, Secret secretKey, boolean useRole, Destination dest, Map<String, String> userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy) {
super(accessKey, secretKey, useRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy);
public S3GzipCallable(String accessKey, Secret secretKey, boolean useRole, Destination dest, Map<String, String> userMetadata, Map<String, String> userTags, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy) {
super(accessKey, secretKey, useRole, dest, userMetadata, userTags, storageClass, selregion, useServerSideEncryption, proxy);
}

// Return a File containing the gzipped contents of the input file.
Expand Down Expand Up @@ -78,10 +79,11 @@ public String invoke(FilePath file) throws IOException, InterruptedException {
// close the stream before the upload has succeeded.
final InputStream gzippedStream = new FileInputStream(localFile);
final ObjectMetadata metadata = buildMetadata(file);
final ObjectTagging tags = s3Tagging();
metadata.setContentEncoding("gzip");
metadata.setContentLength(localFile.length());

upload = Uploads.getInstance().startUploading(getTransferManager(), file, gzippedStream, getDest().bucketName, getDest().objectName, metadata);
upload = Uploads.getInstance().startUploading(getTransferManager(), file, gzippedStream, getDest().bucketName, getDest().objectName, metadata, tags);

Check warning on line 86 in src/main/java/hudson/plugins/s3/callable/S3GzipCallable.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered lines

Lines 28-86 are not covered by tests

String md5 = MD5.generateFromFile(localFile);

Expand Down
11 changes: 7 additions & 4 deletions src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package hudson.plugins.s3.callable;

import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.ObjectTagging;
import hudson.FilePath;
import hudson.ProxyConfiguration;
import hudson.plugins.s3.Destination;
Expand All @@ -14,8 +15,8 @@
public final class S3UploadCallable extends S3BaseUploadCallable implements MasterSlaveCallable<String> {
private static final long serialVersionUID = 1L;

public S3UploadCallable(String accessKey, Secret secretKey, boolean useRole, Destination dest, Map<String, String> userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy) {
super(accessKey, secretKey, useRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy);
public S3UploadCallable(String accessKey, Secret secretKey, boolean useRole, Destination dest, Map<String, String> userMetadata, Map<String, String> userTags, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy) {
super(accessKey, secretKey, useRole, dest, userMetadata, userTags, storageClass, selregion, useServerSideEncryption, proxy);
}

/**
Expand All @@ -24,9 +25,11 @@ public S3UploadCallable(String accessKey, Secret secretKey, boolean useRole, Des
@Override
public String invoke(FilePath file) throws IOException, InterruptedException {
final ObjectMetadata metadata = buildMetadata(file);
final ObjectTagging tags = s3Tagging();

Uploads.getInstance().startUploading(getTransferManager(), file, file.read(), getDest().bucketName, getDest().objectName, metadata);
Uploads.getInstance().startUploading(getTransferManager(), file, file.read(), getDest().bucketName, getDest().objectName, metadata, tags);

Check warning on line 30 in src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered lines

Lines 28-30 are not covered by tests

return MD5.generateFromFile(file);
}
}

}
9 changes: 9 additions & 0 deletions src/main/resources/hudson/plugins/s3/Entry/config.jelly
Original file line number Diff line number Diff line change
Expand Up @@ -49,5 +49,14 @@
</f:entry>
</f:repeatableProperty>
</f:entry>
<f:entry title="Object tags">
<f:repeatableProperty field="userTags">
<f:entry title="">
<div align="right">
<f:repeatableDeleteButton />
</div>
</f:entry>
</f:repeatableProperty>
</f:entry>

</j:jelly>
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,16 @@
</f:repeatableProperty>
</f:entry>

<f:entry title="Object tags">
<f:repeatableProperty field="userTags">
<f:entry title="">
<div align="right">
<f:repeatableDeleteButton />
</div>
</f:entry>
</f:repeatableProperty>
</f:entry>

<f:entry title="Publish Failure Result Constraint" field="pluginFailureResultConstraint">
<f:select />
</f:entry>
Expand Down
11 changes: 11 additions & 0 deletions src/main/resources/hudson/plugins/s3/TagPair/config.jelly
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<?jelly escape-by-default='true'?>
<j:jelly xmlns:j="jelly:core" xmlns:f="/lib/form">

<f:entry title="Tag key" field="key">
<f:textbox />
</f:entry>
<f:entry title="Tag value" field="value">
<f:textbox />
</f:entry>

</j:jelly>
4 changes: 4 additions & 0 deletions src/main/resources/hudson/plugins/s3/TagPair/help-key.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
<div>
Tag key for the files from this build. Can contain macros (e.g. environment
variables).
</div>
4 changes: 4 additions & 0 deletions src/main/resources/hudson/plugins/s3/TagPair/help-value.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
<div>
Tag value for the files from this build. Can contain macros (e.g.
environment variables).
</div>
Loading

0 comments on commit 306ef00

Please sign in to comment.