Merge pull request #53 from Duke-GCB/update_files

Update files

Author: johnbradley

ddsc/core/ddsapi.py

@@ -378,6 +378,19 @@ def create_file(self, parent_kind, parent_id, upload_id):
         }
         return self._post("/files/", data)
 
+    def update_file(self, file_id, upload_id):
+        """
+        Send PUT request to /files/{file_id} to update the file contents to upload_id and sets a label.
+        :param file_id: str uuid of file
+        :param upload_id: str uuid of the upload where all the file chunks where uploaded
+        :param label: str short display label for the file
+        :return: requests.Response containing the successful result
+        """
+        put_data = {
+            "upload[id]": upload_id,
+        }
+        return self._put("/files/" + file_id, put_data, content_type=ContentType.form)
+
     def send_external(self, http_verb, host, url, http_headers, chunk):
         """
         Used with create_upload_url to send a chunk the the possibly external object store.

ddsc/core/fileuploader.py

@@ -56,8 +56,13 @@ def upload(self, project_id, parent_kind, parent_id):
         chunk_processor = self._make_chunk_processor()
         chunk_processor.run()
         self.data_service.complete_upload(self.upload_id)
-        result = self.data_service.create_file(parent_kind, parent_id, self.upload_id)
-        return result.json()['id']
+        if self.local_file.remote_id:
+            file_id = self.local_file.remote_id
+            self.data_service.update_file(file_id, self.upload_id)
+            return file_id
+        else:
+            result = self.data_service.create_file(parent_kind, parent_id, self.upload_id)
+            return result.json()['id']
 
     @staticmethod
     def send_file_external(data_service, url_json, chunk):

ddsc/core/localstore.py

@@ -209,18 +209,10 @@ def update_remote_ids(self, remote_file):
         Based on a remote file try to assign a remote_id and compare hash info.
         :param remote_file: RemoteFile remote data pull remote_id from
         """
-        # Since right now the remote server allows duplicates
-        # this could be called multiple times for the same local file
-        # as long as one matches we have the file uploaded.
-
-        # if we don't have a uuid yet any will do
-        if not self.need_to_send:
-            self.remote_id = remote_file.id
-        # but we prefer the one that matches our hash
+        self.remote_id = remote_file.id
         (alg, file_hash) = self.get_hashpair()
         if alg == remote_file.hash_alg and file_hash == remote_file.file_hash:
             self.need_to_send = False
-            self.remote_id = remote_file.id
 
     def set_remote_id_after_send(self, remote_id):
         """

setup.py

@@ -2,7 +2,7 @@
 
 
 setup(name='DukeDSClient',
-        version='0.2.5',
+        version='0.2.6',
         description='Command line tool(ddsclient) to upload/manage projects on the duke-data-service.',
         url='https://github.com/Duke-GCB/DukeDSClient',
         keywords='duke dds dukedataservice',

test_scripts/inside_docker.sh

@@ -9,7 +9,8 @@ echo "upload_workers: $UPLOAD_WORKERS" >> $CONFIG_FILE
 
 set -e
 echo "Installing ddsclient"
-eval $TEST_PYTHON setup.py install 2>/dev/null >/dev/null
+$TEST_PYTHON setup.py install 2>/dev/null >/dev/null
 
 echo "Running tests"
 $TEST_PYTHON -m unittest discover test_scripts/tests/
+

test_scripts/run.sh

@@ -65,17 +65,22 @@ run_test()
                    -e TEST_PYTHON=$TEST_PYTHON -e DDS_IP=$DDS_IP \
                    -e DDS_USER_KEY=$DDS_USER_KEY -e DDS_AGENT_KEY=$DDS_AGENT_KEY \
                    -v $CUR_DIR/../DukeDSClientData:/tmp/DukeDSClientData \
-                   dds_test | tee -a $TEMPFILE
-    if [ "$?" -ne "0" ]
+                   dds_test > $TEMPFILE
+    RET=$?
+    echo "Test exit status $RET"
+    if [ "$RET" -ne "0" ]
     then
       echo "ERROR: $TEST_PYTHON tests failed"
-      echo "see $TEMP for more"
+      tail -n 10 $TEMPFILE
+      echo "see $TEMPFILE for more"
       exit 1
     fi
     delete_user_data
     rm $TEMPFILE
 }
 
+docker rm $(docker ps -a -q --filter="name=dukedataservice_") 2>/dev/null
+
 build_docker_file
 start_dds
 

test_scripts/tests/test_upload.py

@@ -81,9 +81,26 @@ def test_upload_download_docs(self):
         shutil.rmtree("../my_docs")
 
     def test_upload_download_bigfile(self):
+        """
+        Uploads a really big file. This takes quite a while to run.
+        """
         self.assertUploadWorks("upload -p bigfile /tmp/DukeDSClientData/bigfile.tar")
         self.assertDownloadWorks("download -p bigfile /tmp/bf")
         self.assertFilesSame('/tmp/DukeDSClientData/bigfile.tar', '/tmp/bf/bigfile.tar')
 
+    def test_update_file(self):
+        """
+        Test that we can update the contents of a file after uploading it.
+        """
+        with open("/tmp/abc.txt", "w") as data_file:
+            data_file.write("one line")
+        self.assertUploadWorks("upload -p change_it /tmp/abc.txt")
+        with open("/tmp/abc.txt", "w") as data_file:
+            data_file.write("one line")
+            data_file.write("two line")
+        self.assertUploadWorks("upload -p change_it /tmp/abc.txt")
+        self.assertDownloadWorks("download -p change_it /tmp/change_it")
+        self.assertFilesSame('/tmp/abc.txt', '/tmp/change_it/abc.txt')
+
 if __name__ == '__main__':
     unittest.main()