fix merge

CMakeLists.txt

@@ -28,8 +28,8 @@ set(CMAKE_MODULE_PATH
 )
 
 set(VERSION_MAJOR 3)
-set(VERSION_MINOR 7)
-set(VERSION_PATCH 10)
+set(VERSION_MINOR 8)
+set(VERSION_PATCH 0)
 set(VERSION_STRING ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH})
 
 # Generate include header with the version string

Vagrantfile

@@ -24,6 +24,8 @@ Vagrant.configure("2") do |config|
     config.vm.box = box
     #config.vm.provision "file", source: "packaging/rpm/fts.spec", destination: "/tmp/fts.spec"
     config.vm.provision :shell, path: "bootstrap.sh"
+    config.vm.synced_folder "../gfal2/", "/home/vagrant/gfal2/"
+    config.vm.synced_folder "../gfal2-bindings/", "/home/vagrant/gfal2-bindings/"
 
     config.vm.provider "virtualbox" do |vb|
         vb.memory = 1024

bootstrap.sh

@@ -10,8 +10,9 @@ protect=1
 EOF
 
 yum install -y vim
+yum install -y nano
 yum groupinstall -y 'Development Tools'
-yum install -y epel-release yum-builddep git
+yum install -y epel-release yum-utils git
 yum-builddep -y "/vagrant/packaging/rpm/fts.spec"
 yum install -y gfal2-all
 
@@ -21,5 +22,7 @@ cp "/vagrant/src/config/fts3config" "/etc/fts3/fts3config"
 
 mkdir -p "/var/lib/fts3/"
 mkdir -p "/var/log/fts3"
+mkdir -p "/home/vagrant/gfal2"
+mkdir -p "/home/vagrant/gfal2-bindings"
 chown vagrant.vagrant "/var/lib/fts3/"
 chown vagrant.vagrant "/var/log/fts3/"

cmake/modules/DefineInstallationPaths.cmake

@@ -27,6 +27,10 @@ if (UNIX)
     "${SHARE_INSTALL_PREFIX}/${APPLICATION_NAME}"
     CACHE PATH "The parent directory where applications can install their data")
 
+  SET(DATADIR_INSTALL_PREFIX
+    "${CMAKE_INSTALL_PREFIX}/var/lib/${APPLICATION_NAME}"
+    CACHE PATH "The parent directory where applications can install runtime data")
+
   # The following are directories where stuff will be installed to
   SET(BIN_INSTALL_DIR
     "${EXEC_INSTALL_PREFIX}/bin"

packaging/docker/Dockerfile

@@ -1,14 +1,6 @@
 FROM centos:7
 MAINTAINER CERN
 
-# Add cern repos
-ADD http://linux.web.cern.ch/linux/centos7/CentOS-CERN.repo /etc/yum.repos.d/CentOS-CERN.repo
-ADD http://linuxsoft.cern.ch/cern/centos/7/os/x86_64/RPM-GPG-KEY-CentOS-7 /tmp/RPM-GPG-KEY-cern
-RUN /usr/bin/rpm --import /tmp/RPM-GPG-KEY-cern \
-        && /usr/bin/yum --enablerepo=*-testing clean all \
-        && rm /tmp/RPM-GPG-KEY-cern \
-        && rm -rf /var/cache/yum
-
 # Add EPEL repos
 ADD  http://mirror.switch.ch/ftp/mirror/epel/epel-release-latest-7.noarch.rpm  /tmp/epel-release-7.noarch.rpm
 RUN  yum localinstall /tmp/epel-release-7.noarch.rpm -y \
@@ -39,7 +31,7 @@ RUN /usr/bin/yum install -y voms-config-wlcg  voms-config-vo-dteam
 #install supervisor
 RUN yum install -y supervisor
 
-COPY "supervisord" "/etc/supervisor/conf.d/supervisord.conf"
+COPY "packaging/docker/supervisord" "/etc/supervisor/conf.d/supervisord.conf"
 
 #log folder
 RUN mkdir -p /var/log/fts3 && chown fts3:fts3 /var/log/fts3

packaging/rpm/fts.spec

@@ -9,19 +9,19 @@
 %endif
 
 Name:       fts
-Version:    3.7.10
+Version:    3.8.0
 Release:    1%{?dist}
 Summary:    File Transfer Service V3
 Group:      System Environment/Daemons
 License:    ASL 2.0
 URL:        http://fts3-service.web.cern.ch/
 # The source for this package was pulled from upstream's vcs.  Use the
 # following commands to generate the tarball:
-#  git clone https://gitlab.cern.ch/fts/fts3.git -b master --depth=1 fts-3.7.10
-#  cd fts-3.7.10
-#  git checkout v3.7.10
+#  git clone https://gitlab.cern.ch/fts/fts3.git -b master --depth=1 fts-3.8.0
+#  cd fts-3.8.0
+#  git checkout v3.8.0
 #  cd ..
-#  tar --exclude-vcs -vczf fts-3.7.9.tar.gz fts-3.7.9
+#  tar --exclude-vcs -vczf fts-3.8.0.tar.gz fts-3.8.0
 Source0: %{name}-%{version}.tar.gz
 
 %if 0%{?el5}

qa/Dockerfile

@@ -30,7 +30,7 @@ RUN yum-builddep -y fts-server
 
 # Run dependencies
 RUN yum install -y gfal2-plugin-mock git lcov \
-    lcg-CA supervisor voms-config-vo-dteam voms-clients \
+    ca-policy-egi-core supervisor voms-config-vo-dteam voms-clients \
     python-virtualenv python2-pip
 
 # We also need rest, with a patched config file that allows values to be passed via environment variables

src/bdii-cache-update/CMakeLists.txt

@@ -57,7 +57,7 @@ else ()
 endif ()
 
 install(FILES bdii_cache.xml
-    DESTINATION /var/lib/fts3 
+    DESTINATION ${DATADIR_INSTALL_PREFIX}
 	PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ
 )
 

src/cli/BulkSubmissionParser.cpp

@@ -34,7 +34,7 @@ const std::set<std::string> BulkSubmissionParser::file_tokens =
         ("sources")
         ("destinations")
         ("selection_strategy")
-        ("checksums")
+        ("checksum")
         ("filesize")
         ("metadata")
         ("activity");

src/config/fts3config

@@ -130,6 +130,20 @@ LogLevel=INFO
 #SchedulingInterval = 2
 # In seconds, how often to check for messages. Should be less than CheckStalledTimeout/2
 #MessagingConsumeInterval = 1
+#Enable or disable auto session reuse
+AutoSessionReuse = False
+#Max small file size for session reuse in bytes
+#AutoSessionReuseMaxSmallFileSize = 104857600
+#Max big file size for session reuse in bytes
+#AutoSessionReuseMaxBigFileSize = 1073741824
+#Max number of files per session reuse
+#AutoSessionReuseMaxFiles = 1000
+#Max number of big files  per session reuse
+#AutoSessionReuseMaxBigFiles = 2
+
+#flag to use a default job priority
+#by default it queries the system to honour the priorities specified by the users ( value = 0)
+#UseFixedJobPriority = 0
 
 #flag to use a default job priority
 #by default it queries the system to honour the priorities specified by the users ( value = 0)

src/config/serverconfigreader.cpp

@@ -398,6 +398,27 @@ po::options_description ServerConfigReader::_defineConfigOptions()
         po::value<std::string>( &(_vars["SigKillDelay"]) )->default_value("500"),
         "In milliseconds, delay between graceful SIGTERM and SIGKILL"
     )
+    (   "AutoSessionReuse",
+    	po::value<std::string>( &(_vars["AutoSessionReuse"]) )->default_value("false"),
+	"Enable or disable auto session reuse"
+    )
+
+    (   "AutoSessionReuseMaxSmallFileSize",
+    	po::value<int>()->default_value(104857600),
+	"Max small file size for session reuse in bytes"
+    )
+    (   "AutoSessionReuseMaxBigFileSize",
+	 po::value<int>()->default_value(1073741824),
+	 "Max big file size for session reuse in bytes"
+    )
+    (   "AutoSessionReuseMaxFiles",
+         po::value<int>()->default_value(1000),
+         "Max number of files per session reuse"
+    )
+    (   "AutoSessionReuseMaxBigFiles",
+	 po::value<int>()->default_value(2),
+	 "Max number of big files  per session reuse"
+    )
     (
         "UseFixedJobPriority",
         po::value<std::string>( &(_vars["UseFixedJobPriority"]) )->default_value("0"),

src/db/generic/DbUtils.h

@@ -46,45 +46,6 @@ const int HIGH_THROUGHPUT = 50;
 const int AVG_TRANSFER_DURATION = 15;
 const int MAX_TRANSFER_DURATION = 3600;
 
-
-/*check if it's a multiple replicas job*/
-inline bool is_mreplica(const std::list<SubmittedTransfer>& transfers)
-{
-    // if it has less than 2 pairs it wont be a m-replica
-    if (transfers.size() < 2) return false;
-
-    std::string destSurl = transfers.begin()->destination;
-
-    for (auto iter = transfers.begin(); iter != transfers.end(); ++iter)
-        {
-            if(destSurl != iter->destination)
-                {
-                    return false;
-                }
-        }
-
-    return true;
-}
-
-/*check if it's a m-hop job*/
-inline bool is_mhop(const std::list<SubmittedTransfer>& transfers)
-{
-    // if it has less than 2 pairs it wont be a multi-hop
-    if (transfers.size() < 2) return false;
-
-    auto iter = transfers.begin();
-    auto iter2 = transfers.begin();
-    std::advance(iter2, 1);
-
-    for (; iter2 != transfers.end(); ++iter, ++iter2)
-        {
-            if (iter->destination != iter2->source) return false;
-        }
-
-    return true;
-}
-
-
 /**
  * Returns the current time, plus the difference specified
  * in advance, in UTC time

src/db/generic/GenericDbIfce.h

@@ -101,10 +101,6 @@ class GenericDbIfce
     virtual void getReadyTransfers(const std::vector<QueueId>& queues,
             std::map< std::string, std::list<TransferFile>>& files) = 0;
 
-    /// Get a list of multihop jobs ready to go
-    /// @param[out] files   A map where the key is the VO. The value is a queue of pairs (jobId, list of transfers)
-    virtual void getMultihopJobs(std::map< std::string, std::queue<std::pair<std::string, std::list<TransferFile>>>>& files) = 0;
-
     /// Update the status of a transfer
     /// @param jobId            The job ID
     /// @param fileId           The file ID
@@ -126,9 +122,8 @@ class GenericDbIfce
     /// Update the status of a job
     /// @param jobId            The job ID
     /// @param jobState         The job state
-    /// @param pid              The PID of the fts_url_copy process
     /// @note                   If jobId is empty, the pid will be used to decide which job to update
-    virtual bool updateJobStatus(const std::string& jobId, const std::string& jobState, int pid) = 0;
+    virtual bool updateJobStatus(const std::string& jobId, const std::string& jobState) = 0;
 
     /// Get the credentials associated with the given delegation ID and user
     /// @param delegationId     Delegation ID. See insertCredentialCache
@@ -157,7 +152,7 @@ class GenericDbIfce
     /// @param[out] currentActive   The current number of running transfers is put here
     virtual bool isTrAllowed(const std::string& sourceStorage, const std::string& destStorage, int &currentActive) = 0;
 
-    /// Mark a reuse or multihop job (and its files) as failed
+    /// Mark a reuse job (and its files) as failed
     /// @param jobId    The job id
     /// @param pid      The PID of the fts_url_copy
     /// @param message  The error message
@@ -169,7 +164,8 @@ class GenericDbIfce
     /// @param[out] transfers   An array with the expired transfers. Only jobId, fileId and pid are filled
     virtual void reapStalledTransfers(std::vector<TransferFile>& transfers) = 0;
 
-    /// Set the PID for all the files inside a reuse or multihop job
+
+    /// Set the PID for a transfer
     /// @param jobId    The job id for which the files will be updated
     /// @param pid      The process ID
     /// @note           Transfers within reuse and multihop jobs go all together to a single fts_url_copy process
@@ -181,11 +177,12 @@ class GenericDbIfce
     /// @param[in] bulkSize How many jobs per iteration must be processed
     /// @param[out] nJobs   How many jobs have been moved
     /// @param[out] nFiles  How many files have been moved
-    virtual void backup(int intervalDays, long bulkSize, long* nJobs, long* nFiles) = 0;
+    /// @param[out] nDeletions  How many deletions have been moved
+    virtual void backup(int intervalDays, long bulkSize, long* nJobs, long* nFiles, long* nDeletions) = 0;
 
     /// Mark all the transfers as failed because the process fork failed
     /// @param jobId    The job id for which url copy failed to fork
-    /// @note           This method is used only for reuse and multihop jobs
+    /// @note           This method is used only for reuse jobs
     virtual void forkFailed(const std::string& jobId) = 0;
 
     /// Get the link configuration for the link defined by the source and destination given