diff --git a/docs/TheBook/src/main/markdown/cookbook-clients.md b/docs/TheBook/src/main/markdown/cookbook-clients.md index 39b3d36e7bd..7b4d41bcc17 100644 --- a/docs/TheBook/src/main/markdown/cookbook-clients.md +++ b/docs/TheBook/src/main/markdown/cookbook-clients.md @@ -317,7 +317,7 @@ Usage: Example 22.1. surveying the space tokens available in a directory. ```console-user -srm-get-space-tokens srm://dcache.example.org -srm_protocol_version=2 +srm-get-space-tokens srm://dcache.example.org ``` A successful result: @@ -363,7 +363,6 @@ Example: ```console-user srm-reserve-space \ |-desired_size 2000 \ -|-srm_protocol_version=2 \ |-retention_policy=REPLICA \ |-access_latency=ONLINE \ |-guaranteed_size 1024 \ diff --git a/docs/TheBook/src/main/markdown/reference.md b/docs/TheBook/src/main/markdown/reference.md index 872f166291b..23ab4291a4c 100644 --- a/docs/TheBook/src/main/markdown/reference.md +++ b/docs/TheBook/src/main/markdown/reference.md @@ -6,7 +6,6 @@ Part IV. Reference + [27. dCache Clients](rf-clients-srm.md) - [The SRM Client Suite](rf-clients-srm.md#the-srm-client-suite) - [srmcp](rf-clients-srm.md#srmcp) — Copy a file from or to an SRM or between two SRMs. - - [srmstage](rf-clients-srm.md#srmstage) — Request staging of a file. - [dccp](rf-clients-srm.md#dccp) - [dccp](rf-clients-srm.md#dccp) — Copy a file from or to a dCache server. diff --git a/docs/TheBook/src/main/markdown/rf-clients-srm.md b/docs/TheBook/src/main/markdown/rf-clients-srm.md index 9049fab5495..af36a7113e3 100644 --- a/docs/TheBook/src/main/markdown/rf-clients-srm.md +++ b/docs/TheBook/src/main/markdown/rf-clients-srm.md @@ -6,7 +6,6 @@ Chapter 27. dCache Clients + [The SRM Client Suite](#the-srm-client-suite) [srmcp](#srmcp) — Copy a file from or to an SRM or between two SRMs. - [srmstage](#srmstage) — Request staging of a file. + [dccp](#dccp) [dccp](#dccp) — Copy a file from or to a dCache server. @@ -76,22 +75,6 @@ Example: srmcp --streams_num=1 --server_mode=active sourceUrl destinationUrl ``` -### Description - -### srmstage -srmstage - Request staging of a file. - -### synopsis -srmstage[srmUrl] - -Arguments -srmUrl -The URL of the file which should be staged. - -### Description - -Provides an option to the user to stage files from HSM to dCache and not transfer them to the user right away. This case will be useful if files are not needed right away at user end, but its good to stage them to dcache for faster access later. - ### dccp dccp — Copy a file from or to a dCache server. diff --git a/modules/srm-client/README b/modules/srm-client/README index 0712a057cec..6e87542d82d 100644 --- a/modules/srm-client/README +++ b/modules/srm-client/README @@ -15,9 +15,6 @@ What are the tools available in this package? _____________________________________________ - SRM V1.1 clients: srmcp, srm-get-metadata, srm-get-request-status, - srm-advisory-delete, srmping, srmstage - SRM V2.2 clients: srmcp, srm-reserve-space, srm-release-space, srmrm, srmrmdir, srmls, srmmkdir, srmmv, srmping, srm-bring-online @@ -26,7 +23,7 @@ What is srmcp ------------- - srmcp is the implementation of srm client as specified by srm spec v1.1 and srm spec v2.2 + srmcp is the implementation of srm client as specified by srm spec v2.2 What can I use srmcp for. _______________________ @@ -90,8 +87,6 @@ Usage: srmcp [command line options] source(s) destination a pair of positive integers separated by ":", not set by default -gss_expected_name= default value is host --srm_protocol_version= - or just specify -1 or -2 -conf= default value is config.xml -save_conf= no transfer will be performed if this option is specified @@ -130,13 +125,7 @@ Usage: srmcp [command line options] source(s) destination -pushmode= true to use the push mode in case of srm Mass Storage Systems (MSS) to MSS copy, false to use the pull mode, the default mode is pull mode (false) --srmstage= - if set to true - the source files are staged only onto disk cache - and not transferred to client right away> - if set to false - the source files are transferred to the client -use_urlcopy_script= use java native copiers of use urcopy script --srm_protocol_version=<1 or 2> 1 for srm 1.1 or 2 for srm 2.2, no other protocols are supported - or just specify -1 or -2 -request_lifetime= request lifetime in seconds -copyjobfile= where is the path to the text file containing the list of sources and destination diff --git a/modules/srm-client/src/main/bin/srm-advisory-delete b/modules/srm-client/src/main/bin/srm-advisory-delete deleted file mode 100755 index 9e0797b65b4..00000000000 --- a/modules/srm-client/src/main/bin/srm-advisory-delete +++ /dev/null @@ -1,5 +0,0 @@ -#!@SHELL_PATH@ - -@INIT_SCRIPT@ - -exec "${SRM_PATH}/lib/srm" -advisoryDelete "$@" diff --git a/modules/srm-client/src/main/bin/srm-get-metadata b/modules/srm-client/src/main/bin/srm-get-metadata deleted file mode 100755 index 346decac32d..00000000000 --- a/modules/srm-client/src/main/bin/srm-get-metadata +++ /dev/null @@ -1,5 +0,0 @@ -#!@SHELL_PATH@ - -@INIT_SCRIPT@ - -exec "${SRM_PATH}/lib/srm" -getFileMetaData "$@" diff --git a/modules/srm-client/src/main/bin/srm-get-request-status b/modules/srm-client/src/main/bin/srm-get-request-status deleted file mode 100755 index 474bbc9d5d8..00000000000 --- a/modules/srm-client/src/main/bin/srm-get-request-status +++ /dev/null @@ -1,5 +0,0 @@ -#!@SHELL_PATH@ - -@INIT_SCRIPT@ - -exec "${SRM_PATH}/lib/srm" -getRequestStatus "$@" diff --git a/modules/srm-client/src/main/bin/srm-storage-element-info b/modules/srm-client/src/main/bin/srm-storage-element-info deleted file mode 100755 index ba44648af36..00000000000 --- a/modules/srm-client/src/main/bin/srm-storage-element-info +++ /dev/null @@ -1,5 +0,0 @@ -#!@SHELL_PATH@ - -@INIT_SCRIPT@ - -exec "${SRM_PATH}/lib/srm" -getStorageElementInfo "$@" diff --git a/modules/srm-client/src/main/bin/srmstage b/modules/srm-client/src/main/bin/srmstage deleted file mode 100755 index 5b49fcc368b..00000000000 --- a/modules/srm-client/src/main/bin/srmstage +++ /dev/null @@ -1,5 +0,0 @@ -#!@SHELL_PATH@ - -@INIT_SCRIPT@ - -exec "${SRM_PATH}/lib/srm" -stage "$@" diff --git a/modules/srm-client/src/main/java/gov/fnal/srm/util/Configuration.java b/modules/srm-client/src/main/java/gov/fnal/srm/util/Configuration.java index 1d499219f6e..c3d661f1cc2 100644 --- a/modules/srm-client/src/main/java/gov/fnal/srm/util/Configuration.java +++ b/modules/srm-client/src/main/java/gov/fnal/srm/util/Configuration.java @@ -114,12 +114,10 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING public class Configuration extends ConnectionConfiguration { public static final String SRMCPCONFIGNAMESPACE="srmcp.srm.fnal.gov"; - private static final String webservice_pathv1 = "srm/managerv1"; - private static final String webservice_pathv2 = "srm/managerv2"; private static final String HOME_DIRECTORY=System.getProperty("user.home"); private static final String PATH_SEPARATOR=System.getProperty("file.separator"); private static final String CONFIGURATION_DIRECTORY=".srmconfig"; - private String DEFAULT_CONFIG_FILE=HOME_DIRECTORY+ + private static final String DEFAULT_CONFIG_FILE=HOME_DIRECTORY+ PATH_SEPARATOR+ CONFIGURATION_DIRECTORY+ PATH_SEPARATOR+"config.xml"; @@ -245,6 +243,7 @@ public void setHelp(boolean help) { @Option( name = "webservice_path", description = "path to wsdl in web service url", + defaultValue = "srm/managerv2", required=false, log=true, save=true @@ -252,15 +251,7 @@ public void setHelp(boolean help) { private String webservice_path; public String getWebservice_path() { - String ws_path = webservice_path; - if(ws_path ==null) { - if(srm_protocol_version == 2) { - ws_path = webservice_pathv2; - } else { - ws_path = webservice_pathv1; - } - } - return ws_path; + return webservice_path; } public void setWebservice_path(String webservice_path) { @@ -873,22 +864,6 @@ public void setUse_urlcopy_script(boolean use_urlcopy_script) { this.use_urlcopy_script = use_urlcopy_script; } - - @Option( - name = "getFileMetaData", - description = "gets srm meta data for specified surls", - defaultValue = "false", - required=false, - log=true - ) - private boolean getFileMetaData; - - public boolean isGetFileMetaData() { return getFileMetaData; } - - public void setGetFileMetaData(boolean getFileMetaData) { - this.getFileMetaData = getFileMetaData; - } - @Option( name = "ls", description = "list content of directory", @@ -1243,51 +1218,8 @@ public void setExtendFileLifetimeRequestToken(String token) { public void setExtendFileLifetime(boolean x) { extendFileLifetime=x; } - @Option( - name = "advisoryDelete", - description = "performs AdvisoryDelete", - defaultValue = "false", - required=false, - log=true - ) - private boolean advisoryDelete; - - public boolean isAdvisoryDelete() { return advisoryDelete; } - - public void setAdvisoryDelete(boolean x) { advisoryDelete=x; } - - @Option( - name = "getRequestStatus", - description = "gets Request Status", - defaultValue = "false", - required=false, - log=true - ) - private boolean getRequestStatus; - - public boolean isGetRequestStatus() { return getRequestStatus; } - - public void setGetRequestStatus(boolean x) { getRequestStatus=x; } - private String getRequestStatusSurl; - @Option( - name = "request_id", - description = "request token", - defaultValue = "0", - required=false, - log=true - ) - private int getRequestStatusId; - - public int getGetRequestStatusId() { - return getRequestStatusId; - } - - public void setGetRequestStatusId(int getRequestStatusId) { - this.getRequestStatusId = getRequestStatusId; - } - @Option( name = "connect_to_wsdl", description = "connect to WSDL instead of connecting to the server directly", @@ -1383,23 +1315,6 @@ public void setStorageType(String storage_type) { this.storagetype = storage_type; } - @Option( - name = "stage", - description = "performs srm \"get\", without actual copy of the files with the hope that this will trigger staging in srm managed", - defaultValue = "false", - required=false, - log=true - ) - private boolean stage; - - public boolean isStage() { - return stage; - } - - public void setStage(boolean stage) { - this.stage = stage; - } - @Option( name = "l", description = "long format mode", @@ -1489,42 +1404,6 @@ public void setLsCount(Integer count) { lsCount=count; } - @Option( - name = "srm_protocol_version", - description = "srm protocol version", - defaultValue = "0", - required=false, - log=true, - save=true - ) - private int srm_protocol_version; - - public int getSrmProtocolVersion() { - return srm_protocol_version; - } - - public void setSrmProtocolVersion(int srmProtocolVersion) { - this.srm_protocol_version = srmProtocolVersion; - } - - @Option( - name = "1", - description = "srm protocol version 1", - defaultValue = "false", - required=false, - log=false - ) - private boolean isSrmv1; - - @Option( - name = "2", - description = "srm protocol version 2", - defaultValue = "false", - required=false, - log=false - ) - private boolean isSrmv2; - @Option( name = "request_lifetime", description = " request lifetime", @@ -1794,9 +1673,6 @@ public int getNextByteTimeout() { + " [-extraInfo=: [-extraInfo=:...]]\n" + "\t\t Applies to files only.\n"; - private String stage_options = - " stage options: None \n"; - private String getPermission_options = " srm-get-permissions options: \n" + " [-extraInfo=: [-extraInfo=:...]]\n"; @@ -1823,9 +1699,6 @@ public final String usage() { "x509_user_trusted_certificates", "globus_tcp_port_range", "gss_expected_name", - "srm_protocol_version", - "1", - "2", "conf", "save_conf", "retry_timeout", @@ -1836,13 +1709,6 @@ public final String usage() { "transport", "h", "help"); - if (getFileMetaData) { - return - "\nUsage:get-file-metadata [command line options] srmurl\n\n"+ - " default options can be set in configuration file \n"+ - " or overriden by the command line options\n\n"+ - general_options; - } if(getSpaceTokens) { String getSpaceTokensOptions="get-space-tokens options :\n"+ OptionParser.printOptions(this,"space_desc"); @@ -1852,24 +1718,6 @@ public final String usage() { " or overriden by the command line options\n\n"+ (isHelp()==true?general_options+getSpaceTokensOptions:getSpaceTokensOptions); } - if (advisoryDelete) { - return - "Usage:advisory-delete [command line options] srmurl(s)\n"+ - " default options can be set in configuration file \n"+ - " or overriden by the command line options\n\n"+ - general_options; - } - if (getRequestStatus) { - String getRequestStatusOptions=" get-request-status options :\n"+ - OptionParser.printOptions(this,"request_id"); - return - "\nUsage:get-request-status [command line options] srmurl \n\n"+ - " where srmurl is one of the surl specified in the original request\n" + - " and is used for determening the srm endpoint location\n"+ - " default options can be set in configuration file \n"+ - " or overriden by the command line options\n\n"+ - (isHelp()==true?general_options+getRequestStatusOptions:getRequestStatusOptions); - } if (copy) { String copy_options=" srmcp options :\n"+ OptionParser.printOptions(this,"urlopy", @@ -1888,11 +1736,7 @@ public final String usage() { "connection_type", "overwrite_mode", "pushmode", - "srmstage", "use_urlcopy_script", - "srm_protocol_version", - "2", - "1", "priority", "request_lifetime", "copyjobfile", @@ -1935,8 +1779,6 @@ public final String usage() { "array_of_client_networks", "retention_policy", "access_latency", - "srm_protocol_version", - "2","1", "request_lifetime", "lifetime", "priority", @@ -2089,14 +1931,6 @@ public final String usage() { " the command line options are one or more of the following:\n"+ (isHelp()==true?general_options+mkdir_options:mkdir_options); } - if(stage) { - return - "\nUsage: srmstage [command line options] srmUrl [[srmUrl]...] \n\n" + - " default options can be set in configuration file \n"+ - " or overriden by the command line options\n\n"+ - (isHelp()==true?general_options+stage_options:stage_options); - - } if(getPermission) { return "\nUsage: srmgetpermission [command line options] srmUrl [[srmUrl]...] \n\n" + @@ -2152,9 +1986,6 @@ public final String usage() { " -copy performs srm \"get\", \"put\", or \"copy\" depending \n"+ " on arguments \n"+ " -bringOnline performs srmBringOnline \n"+ - " -stage performs srm \"get\", without actual copy of the files \n"+ - " with the hope that this will trigger staging in srm managed\n"+ - " hierarchical storage system\n"+ " -mv performs srm \"mv\" of files and directories \n"+ " -ls list content of directory \n"+ " -rm remove file(s) \n"+ @@ -2163,21 +1994,16 @@ public final String usage() { " -getPermissions get file(s) permissions \n"+ " -checkPermissions check file permissions \n"+ " -setPermissions set file permissions \n"+ - " -extendFileLietime extend lifetime of existing SURL(s) pf volatile and durable files \n"+ + " -extendFileLifetime extend lifetime of existing SURL(s) pf volatile and durable files \n"+ " or pinned lifetime of TURL(s)\n"+ - " -advisoryDelete performs srm advisory delete \n"+ " -abortRequest to abort request. \n"+ " -abortFiles to abort files. \n"+ " -releaseFiles to unpin files. \n"+ - " -getRequestStatus obtains and prints srm request status \n"+ " -getRequestSummary is to retrieve a summary of the previously submitted request. \n"+ - " -getGetRequestTokens retrieves request tokens for the clients requests, given client provided request description.\n"+ + " -getRequestTokens retrieves request tokens for the clients requests, given client provided request description.\n"+ " This is to accommodate lost request tokens. This can also be used for getting all request tokens.\n"+ - " -getFileMetaData gets srm meta data for specified surls\n"+ " -getSpaceTokens gets space tokens belonging to this user\n"+ " which have a corresponding description (if provided)\n"+ - " -getStorageElementInfo prints storage element info, in particular\n"+ - " it prints total, availabale and used space amount\n"+ " -reserveSpace performs explicit space reservation \n"+ " -releaseSpace performs release of explicit space reservation \n"+ " -getSpaceMetaData retrieves and prints metadata for given space tokens\n"+ @@ -2399,26 +2225,6 @@ else if (!value.startsWith("=")) { throw new IllegalArgumentException("illegal number of retries : "+ getRetry_num()); } - if (isSrmv1&&isSrmv2) { - throw new IllegalArgumentException( - "only one option of -srm_protocol_version, -1 or -2 should be specified"); - } - if ((isSrmv1||isSrmv2) && args.hasOption("srm_protocol_version")) { - throw new IllegalArgumentException( - "only one option of -srm_protocol_version, -1 or -2 should be specified"); - } - if (isSrmv1) { - srm_protocol_version=1; - } - else if (isSrmv2) { - srm_protocol_version=2; - } - else if (srm_protocol_version==0) { - srm_protocol_version=2; - } - if(srm_protocol_version != 1 && srm_protocol_version != 2) { - throw new IllegalArgumentException("illegal srm protocol version "+ srm_protocol_version); - } if(request_lifetime <= 0 && request_lifetime != -1) { throw new IllegalArgumentException("illegal value for request lifetime"+ request_lifetime); @@ -2441,10 +2247,6 @@ else if (srm_protocol_version==0) { copy ^ bringOnline ^ ping ^ - getFileMetaData ^ - advisoryDelete ^ - getRequestStatus ^ - stage ^ getPermission ^ checkPermission ^ setPermission ^ @@ -2473,42 +2275,7 @@ else if (srm_protocol_version==0) { surls = args.getArguments().toArray(String[]::new); srmUrl = new URI(args.argv(0)); - // - // take care of protocol version for srm v2.2 functions - // (override whatever user has specified) - // - if (is_AbortRequest|| - reserveSpace || - checkPermission || - is_AbortFiles || - is_rm || - is_mv || - setPermission || - getSpaceTokens || - is_ReleaseFiles || - is_mkdir || - getSpaceMetaData || - ls || - bringOnline || - releaseSpace || - extendFileLifetime || - is_getRequestTokens || - getPermission || - is_getRequestSummary || - is_rmdir) { - srm_protocol_version=2; - } - if (getRequestStatus) { - if (numberOfArguments == 1) { - getRequestStatusSurl = args.argv(0); - } - else { - throw new IllegalArgumentException( - "one and only one storage element info server " + - "wsdl url should be specified"); - } - } - else if (is_getRequestTokens) { + if (is_getRequestTokens) { getRequestStatusSurl = args.argv(0); } else if (is_getRequestSummary) { @@ -2591,9 +2358,6 @@ private void readCopyOptions() { protocols = readListOfOptions(protocols_list,","); arrayOfClientNetworks = readListOfOptions(array_of_client_networks,","); - if (spaceToken!=null) { - srm_protocol_version = 2; - } readCksmOptions(); } @@ -2792,13 +2556,6 @@ public String toString() { } } } - if (getFileMetaData && surls != null) { - sb.append("\n\taction is getFileMetaData"); - for (int i = 0; i fileIDs = new HashSet<>(); - private HashMap fileIDsMap = new HashMap<>(); - private int requestID; - private Thread hook; - - public SRMCopyClientV1(Configuration configuration, java.net.URI[] from, java.net.URI[] to) { - super(configuration); - report = new Report(from,to,configuration.getReport()); - this.from = from; - this.to = to; - } - - @Override - public void connect() throws Exception { - if ( configuration.isPushmode() ) { - dsay("starting transfer in push mode"); - connect(from[0]); - } - else { - dsay("starting transfer in pull mode"); - connect(to[0]); - } - } - - @Override - public void start() throws Exception { - int len = from.length; - String[] srcSURLs = new String[len]; - String[] dstSURLs = new String[len]; - boolean[] wantPerm = new boolean[len]; - // do it temporarily to avoid - // permanent writes at jlab - Arrays.fill(wantPerm,false); - for(int i = 0; i fileStatus with fileId="+ - rs.fileStatuses[i].fileId+ - " state =="+rs.fileStatuses[i].state); - } - throw new IOException("Request with requestId ="+rs.requestId+ - " rs.state = "+rs.state+" rs.error = "+rs.errorMessage); - } - - if(rs.fileStatuses.length != len) { - esay( "incorrect number of RequestFileStatuses"+ - "in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - throw new IOException("incorrect number of RequestFileStatuses "+ - "in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - } - - for(int i =0; i iter = fileIDs.iterator(); - Collection removeIDs = new HashSet<>(); - while(iter.hasNext()) { - Integer nextID = iter.next(); - RequestFileStatus frs = getFileRequest(rs,nextID); - if(frs == null) { - throw new IOException("request status does not have"+ - "RequestFileStatus fileID = "+nextID); - } - - if(frs.state.equals("Failed")) { - say("FileRequestStatus is Failed => copying of "+frs.SURL+ - " has failed"); - setReportFailed(new java.net.URI(frs.SURL),new java.net.URI(frs.TURL), "copy failed" +rs.errorMessage); - removeIDs.add(nextID); - } - - if(frs.state.equals("Ready") ) { - say("FileRequestStatus is Ready => copying of "+frs.SURL+ - " is complete"); - setReportSucceeded(new java.net.URI(frs.SURL),new java.net.URI(frs.TURL)); - removeIDs.add(nextID); - srm.setFileStatus(requestID, nextID,"Done"); - } - if(frs.state.equals("Done")) { - say("FileRequestStatus fileID = "+nextID+" is Done => copying of "+frs.SURL+ - " is complete"); - setReportSucceeded(new java.net.URI(frs.SURL),new java.net.URI(frs.TURL)); - removeIDs.add(nextID); - } - } - fileIDs.removeAll(removeIDs); - - if(fileIDs.isEmpty()) { - Runtime.getRuntime().removeShutdownHook(hook); - //we have copied all files - break; - } - - try { - int retrytime = rs.retryDeltaTime; - if( retrytime <= 0 ) { - retrytime = 1; - } - - say("sleeping "+retrytime+" seconds ..."); - Thread.sleep(retrytime * 1000); - } - catch(InterruptedException ie) { - } - - rs = srm.getRequestStatus(requestID); - if(rs == null) { - throw new IOException(" null requests status"); - } - - if(rs.state.equals("Failed")) { - for (Integer nextID1 : fileIDs) { - RequestFileStatus frs = getFileRequest(rs, nextID1); - if (frs.state.equals("Failed")) { - say("FileRequestStatus is Failed => copying of " + frs.SURL + - " has failed"); - setReportFailed(new java.net.URI(frs.SURL), new java.net.URI(frs.TURL), "copy failed" + rs.errorMessage); - } - if (frs.state.equals("Ready")) { - say("FileRequestStatus fileID = " + nextID1 + " is Ready => copying of " + frs.SURL + - " is complete"); - setReportSucceeded(new java.net.URI(frs.SURL), new java.net.URI(frs.TURL)); - } - if (frs.state.equals("Done")) { - say("FileRequestStatus fileID = " + nextID1 + " is Done => copying of " + frs.SURL + - " is complete"); - setReportSucceeded(new java.net.URI(frs.SURL), new java.net.URI(frs.TURL)); - } - } - throw new IOException("Request with requestId ="+rs.requestId+ - " rs.state = "+rs.state+" rs.error = "+rs.errorMessage); - } - if(rs.fileStatuses.length != len) { - esay( "incorrect number of RequestFileStatuses"+ - " in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - throw new IOException("incorrect number of RequestFileStatuses"+ - " in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - } - } - } - catch(IOException ioe) { - if(configuration.isDebug()) { - ioe.printStackTrace(); - } - else { - esay(ioe.toString()); - } - done(rs,srm); - throw ioe; - } - finally { - report.dumpReport(); - if(!report.everythingAllRight()){ - report.reportErrors(System.err); - System.exit(1); - } - } - } - - @Override - public void run() { - say("setting all remaining file statuses of request"+ - " requestId="+requestID+" to \"Done\""); - while(true) { - if(fileIDs.isEmpty()) { - break; - } - Integer fileId = fileIDs.iterator().next(); - fileIDs.remove(fileId); - say("setting file request "+fileId+" status to Done"); - RequestFileStatus rfs = fileIDsMap.get(fileId); - srm.setFileStatus(requestID,rfs.fileId,"Done"); - } - say("set all file statuses to \"Done\""); - } -} - diff --git a/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMDispatcher.java b/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMDispatcher.java index 0af8604f4b1..1694eef995c 100644 --- a/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMDispatcher.java +++ b/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMDispatcher.java @@ -94,8 +94,8 @@ public class SRMDispatcher { public static final int DIRECTORY_URL=0x80; public static final int UNKNOWN_URL=0x100; private SRMClient srmclient; - private Configuration configuration; - private Logger logger; + private final Configuration configuration; + private final Logger logger; private SRMDispatcher(Configuration configuration) { @@ -152,14 +152,6 @@ public static final void main(String[] args) throws Exception { logger.elog(conf.usage()); System.exit(1); } - if (conf.isGetFileMetaData()) { - if(conf.getGetFileMetaDataSurls() == null || - conf.getGetFileMetaDataSurls().length == 0 ) { - System.err.println( - "cannot get metadata - surls not specified"); - System.exit(1); - } - } else if (conf.isGetPermission()){ if(conf.getGetPermissionSurls() == null || conf.getGetPermissionSurls().length == 0 ) { @@ -289,20 +281,6 @@ else if (conf.isSetPermission()){ } } } - else if (conf.isAdvisoryDelete()) { - if (conf.getAdvisoryDeleteSurls() == null || - conf.getAdvisoryDeleteSurls().length == 0 ) { - System.err.println( - "cannot perform advisory delete - surls not specified"); - System.exit(1); - } - } - else if (conf.isGetRequestStatus()) { - if (conf.getGetRequestStatusSurl() == null ) { - System.err.println("surl, to be used for location of srm, is not specified"); - System.exit(1); - } - } else if (conf.isGetRequestSummary()) { if (conf.getGetRequestStatusSurl() == null ) { System.err.println("surl, to be used for location of srm, is not specified"); @@ -413,18 +391,7 @@ else if (conf.isGetSpaceTokens()) { } private void work() throws Exception { - if (configuration.isGetFileMetaData()) { - String[] surl_strings = configuration.getGetFileMetaDataSurls(); - int number_of_surls = surl_strings.length; - java.net.URI[] surls = new java.net.URI[number_of_surls]; - for(int i=0;i fileIDs = new HashSet<>(); - private HashMap fileIDsMap = new HashMap<>(); - private Copier copier; - private int requestID; - private Thread hook; - /** Creates a new instance of SRMGetClient */ - public SRMGetClientV1(Configuration configuration, java.net.URI[] from, java.net.URI[] to) { - super(configuration); - report = new Report(from,to,configuration.getReport()); - this.protocols = configuration.getProtocols(); - this.from = from; - this.to = to; - } - - @Override - public void connect() throws Exception { - connect(from[0]); - } - - public void setProtocols(String[] protocols) { - this.protocols = protocols; - } - - @Override - public void start() throws Exception { - try { - copier = new Copier(urlcopy,configuration); - copier.setDebug(debug); - new Thread(copier).start(); - int len = from.length; - String SURLS[] = new String[len]; - for(int i = 0; i < len; ++i) { - SURLS[i] = from[i].toASCIIString(); - } - hook = new Thread(this); - Runtime.getRuntime().addShutdownHook(hook); - RequestStatus rs = srm.get(SURLS,protocols); - - if(rs == null) { - throw new IOException(" null requests status"); - } - requestID = rs.requestId; - dsay(" srm returned requestId = "+rs.requestId); - try { - if(rs.state.equals("Failed")) { - esay("rs.state = "+rs.state+" rs.error = "+rs.errorMessage); - for(int i = 0; i< rs.fileStatuses.length;++i) { - edsay(" ====> fileStatus state =="+rs.fileStatuses[i].state); - } - throw new IOException("rs.state = "+rs.state+" rs.error = "+rs.errorMessage); - } - - if(rs.fileStatuses.length != len) { - esay( "incorrect number of RequestFileStatuses"+ - "in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - throw new IOException("incorrect number of RequestFileStatuses"+ - "in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - } - - for(int i =0; i iter = fileIDs.iterator(); - Collection removeIDs = new HashSet<>(); - - while(iter.hasNext()) { - Integer nextID = iter.next(); - RequestFileStatus frs = getFileRequest(rs,nextID); - if(frs == null) { - throw new IOException("request status does not have"+"RequestFileStatus fileID = "+nextID); - } - if(frs.state.equals("Failed")) { - removeIDs.add(nextID); - java.net.URI surl = new java.net.URI(frs.SURL); - java.net.URI filedest = null; - if(len == 1) { - // in case of one file there could be no correspondence between source and destination - filedest = to[0]; - }else { - for(int i =0; i< len;++i) { - if(surl.equals(from[i])) { - filedest = to[i]; - break; - } - } - } - setReportFailed(surl,filedest, rs.errorMessage); - esay( "copying from SURL "+frs.SURL+" to file "+filedest +" failed: File Status is \"Failed\""); - continue; - } - - if(frs.state.equals("Ready") ) { - if(frs.TURL == null) { - throw new IOException(" TURL not found, (check root path in kpwd) fileStatus state =="+frs.state); - } - say("FileRequestStatus with SURL="+frs.SURL+" is Ready"); - say(" received TURL="+ frs.TURL); - java.net.URI globusTURL = new java.net.URI(frs.TURL); - java.net.URI filedest = null; - java.net.URI surl = new java.net.URI(frs.SURL); - - if(len == 1) { - // in case of one file there could be no correspondence between source and destination - filedest = to[0]; - }else { - for(int i =0; i< len;++i) { - if(surl.equals(from[i])) { - filedest = to[i]; - break; - } - } - } - setReportFailed(surl,filedest, "received TURL, but did not complete transfer"); - if(filedest == null) { - String error ="could not find file destination "+"for source SURL "+ surl; - esay( error); - throw new IOException(error); - } - CopyJob job = new SRMV1CopyJob(globusTURL,filedest,srm,requestID, nextID,logger,surl,true,this); - copier.addCopyJob(job); - removeIDs.add(nextID); - } - } - fileIDs.removeAll(removeIDs); - - if(fileIDs.isEmpty()) { - dsay("fileIDs is empty, breaking the loop"); - Runtime.getRuntime().removeShutdownHook(hook); - break; - } - - try{ - int retrytime = rs.retryDeltaTime; - if( retrytime <= 0 ) { - retrytime = 1; - } - say("sleeping "+retrytime+" seconds ..."); - Thread.sleep(retrytime * 1000); - }catch(InterruptedException ie) { - } - - rs = srm.getRequestStatus(requestID); - if(rs == null) { - throw new IOException(" null requests status"); - } - if(rs.state.equals("Failed")) { - esay("rs.state = "+rs.state+" rs.error = "+rs.errorMessage); - for(int i = 0; i< rs.fileStatuses.length;++i) { - edsay(" ====> fileStatus state =="+rs.fileStatuses[i].state); - } - throw new IOException("rs.state = "+rs.state+" rs.error = "+rs.errorMessage); - } - if(rs.fileStatuses.length != len) { - esay( "incorrect number of RequestFileStatuses"+"in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - throw new IOException("incorrect number of RequestFileStatuses"+"in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - } - - for(int i =0; i fileIDs = new HashSet<>(); - private HashMap fileIDsMap = new HashMap<>(); - private Copier copier; - private int requestID; - private Thread hook; - /** Creates a new instance of SRMPutClient */ - public SRMPutClientV1(Configuration configuration, java.net.URI[] from, java.net.URI[] to) { - super(configuration); - report = new Report(from,to,configuration.getReport()); - this.protocols = configuration.getProtocols(); - this.from = from; - this.to = to; - } - - - public void setProtocols(String[] protocols) { - this.protocols = protocols; - } - - @Override - public void connect() throws Exception { - connect(to[0]); - } - - @Override - public void start() throws Exception { - try { - copier = new Copier(urlcopy,configuration); - copier.setDebug(debug); - new Thread(copier).start(); - int len = from.length; - String sources[] = new String[len]; - long sizes[] = new long[len]; - boolean[] wantperm = new boolean[len]; - - Arrays.fill(wantperm,true); - String dests[] = new String[len]; - for(int i = 0; i fileStatus state =="+rs.fileStatuses[i].state); - } - throw new IOException("rs.state = "+rs.state+" rs.error = "+rs.errorMessage); - } - - if(rs.fileStatuses.length != len) { - esay( "incorrect number of RequestFileStatuses"+ - "in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - throw new IOException("incorrect number of RequestFileStatuses"+ - "in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - } - - for(int i =0; i iter = fileIDs.iterator(); - Collection removeIDs = new HashSet<>(); - while(iter.hasNext()) { - Integer nextID = iter.next(); - RequestFileStatus frs = getFileRequest(rs,nextID); - if(frs == null) { - throw new IOException("request status does not have"+"RequestFileStatus fileID = "+nextID); - } - - if(frs.state.equals("Failed")) { - removeIDs.add(nextID); - java.net.URI surl = new java.net.URI(frs.SURL); - java.net.URI filesource = null; - if(len == 1) { - // in case of one file there could be no correspondence between source and destination - filesource = from[0]; - }else { - for(int i =0; i< len;++i) { - if(surl.equals(to[i])) { - filesource = from[i]; - break; - } - } - } - setReportFailed(filesource,surl, rs.errorMessage); - esay( "copying from file file "+filesource+ " to SURL "+frs.SURL +" failed: File Status is \"Failed\""); - continue; - } - if(frs.state.equals("Ready") ) { - if(frs.TURL == null) { - throw new IOException(" TURL not found (check root path in kpwd), fileStatus state =="+frs.state); - } - say("FileRequestStatus with SURL="+frs.SURL+" is Ready"); - say(" received TURL="+ frs.TURL); - java.net.URI globusTURL = new java.net.URI(frs.TURL); - java.net.URI filesource = null; - java.net.URI surl = new java.net.URI(frs.SURL); - if(len == 1) { - // in case of one file there could be no correspondence between source and destination - filesource = from[0]; - }else { - for(int i =0; i< len;++i) { - if(surl.equals(to[i])) { - filesource = from[i]; - break; - } - } - } - //why is setReportFailed Called Here ?? - setReportFailed(filesource,surl, "received TURL, but did not complete transfer"); - if(filesource == null) { - esay("could not find file source for destination SURL " + surl); - throw new IOException("could not find source for destination SURL " + surl); - } - CopyJob job = new SRMV1CopyJob(filesource,globusTURL,srm,requestID, nextID,logger,surl,false,this); - copier.addCopyJob(job); - removeIDs.add(nextID); - } - } - fileIDs.removeAll(removeIDs); - - if(fileIDs.isEmpty()) { - Runtime.getRuntime().removeShutdownHook(hook); - //we are copying all files - break; - } - try { - int retrytime = rs.retryDeltaTime; - if( retrytime <= 0 ) { - retrytime = 1; - } - say("sleeping "+retrytime+" seconds ..."); - Thread.sleep(retrytime * 1000); - }catch(InterruptedException ie) { - } - - rs = srm.getRequestStatus(requestID); - if(rs == null) { - throw new IOException(" null requests status"); - } - - if(rs.state.equals("Failed")) { - esay("rs.state = "+rs.state+" rs.error = "+rs.errorMessage); - for(int i = 0; i< rs.fileStatuses.length;++i) { - edsay(" ====> fileStatus state =="+rs.fileStatuses[i].state); - } - throw new IOException("rs.state = "+rs.state+" rs.error = "+rs.errorMessage); - } - if(rs.fileStatuses.length != len) { - esay( "incorrect number of RequestFileStatuses"+ - "in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - throw new IOException("incorrect number of RequestFileStatuses"+"in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - } - - for(int i =0; i fileIDs = new HashSet<>(); - private HashMap fileIDsMap = new HashMap<>(); - private int requestID; - private Thread hook; - - /** Creates a new instance of SRMStageClient */ - public SRMStageClientV1(Configuration configuration, java.net.URI[] from) { - super(configuration); - report = new Report(from,from,configuration.getReport()); - this.protocols = configuration.getProtocols(); - this.from = from; - } - - @Override - public void connect() throws Exception { - connect(from[0]); - } - - public void setProtocols(String[] protocols) { - this.protocols = protocols; - } - - @Override - public void start() throws Exception { - try { - int len = from.length; - String SURLS[] = new String[len]; - for(int i = 0; i < len; ++i) { - SURLS[i] = from[i].toASCIIString(); - } - hook = new Thread(this); - Runtime.getRuntime().addShutdownHook(hook); - RequestStatus rs = srm.get(SURLS,protocols); - if(rs == null) { - throw new IOException(" null requests status"); - } - requestID = rs.requestId; - dsay(" srm returned requestId = "+rs.requestId); - try { - if(rs.state.equals("Failed")) { - esay("rs.state = "+rs.state+" rs.error = "+rs.errorMessage); - for(int i = 0; i< rs.fileStatuses.length;++i) { - edsay(" ====> fileStatus state =="+rs.fileStatuses[i].state); - } - throw new IOException("rs.state = "+rs.state+" rs.error = "+rs.errorMessage); - } - - if(rs.fileStatuses.length != len) { - esay( "incorrect number of RequestFileStatuses"+ - "in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - throw new IOException("incorrect number of RequestFileStatuses"+ - "in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - } - - for(int i =0; i iter = fileIDs.iterator(); - Collection removeIDs = new HashSet<>(); - - while(iter.hasNext()) { - Integer nextID = iter.next(); - RequestFileStatus frs = getFileRequest(rs,nextID); - if(frs == null) { - throw new IOException("request status does not have"+"RequestFileStatus fileID = "+nextID); - } - if(frs.state.equals("Failed")) { - removeIDs.add(nextID); - java.net.URI surl = new java.net.URI(frs.SURL); - setReportFailed(surl,surl, rs.errorMessage); - esay( "staging of SURL "+frs.SURL+" failed: File Status is \"Failed\""); - continue; - } - - if(frs.state.equals("Ready") ) { - say(frs.SURL+" is staged successfully "); - java.net.URI surl = new java.net.URI(frs.SURL); - removeIDs.add(nextID); - srm.setFileStatus(rs.requestId, nextID,"Done"); - setReportSucceeded(surl,surl); - } - } - - fileIDs.removeAll(removeIDs); - - if(fileIDs.isEmpty()) { - dsay("fileIDs is empty, breaking the loop"); - Runtime.getRuntime().removeShutdownHook(hook); - break; - } - - try{ - int retrytime = rs.retryDeltaTime; - if( retrytime <= 0 ) { - retrytime = 1; - } - say("sleeping "+retrytime+" seconds ..."); - Thread.sleep(retrytime * 1000); - }catch(InterruptedException ie) { - } - - rs = srm.getRequestStatus(requestID); - if(rs == null) { - throw new IOException(" null requests status"); - } - - if(rs.state.equals("Failed")) { - esay("rs.state = "+rs.state+" rs.error = "+rs.errorMessage); - for(int i = 0; i< rs.fileStatuses.length;++i) { - edsay(" ====> fileStatus state =="+rs.fileStatuses[i].state); - } - throw new IOException("rs.state = "+rs.state+" rs.error = "+rs.errorMessage); - } - if(rs.fileStatuses.length != len) { - esay( "incorrect number of RequestFileStatuses"+"in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - throw new IOException("incorrect number of RequestFileStatuses"+"in RequestStatus expected "+len+" received "+rs.fileStatuses.length); - } - - for(int i =0; i