gdcc · qqmyers · Jun 4, 2024 · Jun 4, 2024 · Jun 4, 2024 · Jun 5, 2024
diff --git a/localinstall.sh b/localinstall.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+echo Creating version of DVWebloader that uses only local libraries
+echo Run this script in the directory with the dvwebloader.html file
+
+
+dirLocal=$(pwd)
+
+echo Downloading local copies of remote JavaScript libraries:
+sed -n 's/.*src="\(http[^"]*\)".*/\1/p' *.html | sort -u | sed -n 's/^\(.*\/\)*\(.*\)/sed -i \x27s,\0\,lib\/\2,\x27 *.html/p' > replace_js.sh
+sed -n 's/.*src="\(http[^"]*\)".*/\1/p' *.html | sort -u  > urls_js.txt
+source replace_js.sh
+cat urls_js.txt
+
+echo Downloading local copies of remote JavaScript libraries referenced in js files:
+cd ./js
+sed -n 's/.*src[ ]*=[ ]*"\(http[^"]*\)".*/\1/p' *.js | sort -u | sed -n 's/^\(.*\/\)*\(.*\)/sed -i \x27s,\0\,lib\/\2,\x27 *.js/p' > replacejs_js.sh
+sed -n 's/.*src[ ]*=[ ]*"\(http[^"]*\)".*/\1/p' *.js | sort -u  > urlsjs_js.txt
+source replacejs_js.sh
+cat urlsjs_js.txt
+cd ..
+
+echo Downloading local copies of remote CSS files:
+sed -n 's/.*<link.*href="\(http[^"]*\)".*/\1/p' *.html | sort -u | sed -n 's/^\(.*\/\)*\(.*\)/sed -i \x27s,\0\,lib\/\2,\x27 *.html/p' > replace_css.sh
+sed -n 's/.*<link.*href="\(http[^"]*\)".*/\1/p' *.html | sort -u > urls_css.txt
+source replace_css.sh
+cat urls_css.txt
+
+if [ ! -d ./lib ]; then
+  mkdir ./lib
+fi
+cd ./lib
+while read url; do
+    wget --quiet $url
+done < "../urls_js.txt"
+while read url; do
+    wget --quiet $url
+done < "../js/urlsjs_js.txt"
+
+
+cd ".."
+if [ ! -d ./css ]; then
+  mkdir ./css
+fi
+cd ./css
+while read url; do
+    wget --quiet $url
+done < "../urls_css.txt"
+
+cd ..
+
+
+echo Cleaning Up...
+rm urls_js.txt
+rm urls_css.txt
+rm replace_js.sh
+rm replace_css.sh
+rm js/urlsjs_js.txt
+rm js/replacejs_js.sh
+
+echo Done
+exit 0
diff --git a/src/css/dvwebloader.css b/src/css/dvwebloader.css
@@ -45,6 +45,10 @@ label {
     background-color: beige;
     padding: 10px;
 }
+.warn {
+   background-color: bisque;
+    padding: 10px;
+}
 .file-exists {
     background-color: lightblue;
 }
@@ -67,3 +71,7 @@ h1 {
     width: 58%;
     display:inline-block;
 }
+.badchars {
+   background-color: bisque;
+    padding: 10px;
+}
diff --git a/src/dvwebloader.html b/src/dvwebloader.html
@@ -3,8 +3,8 @@
     <head>
         <title>Dataverse WebLoader</title>
         <link type="text/css" rel="stylesheet" href="css/dvwebloader.css">
-        <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.0.0/core.js"></script>
+        <script src="https://code.jquery.com/jquery-3.7.1.min.js"></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.2.0/core.js"></script>
         <script type="module" src="js/fileupload2.js"></script>
         <script src="js/logoHandler.js"></script>
 
@@ -26,7 +26,7 @@ <h1><span id="title-text">Folder Upload</span></h1>
             </div>
             <div id="filelist"></div>
             <div id="credit">
-                <a href='https://github.com/gdcc/dvwebloader' target='_blank'>DVWebloader v0.2</a><span id="sponsor-text">, development sponsored by UiT/DataverseNO</span>
+                <a href='https://github.com/gdcc/dvwebloader' target='_blank'>DVWebloader v0.3</a><span id="sponsor-text">, development sponsored by UiT/DataverseNO</span>
             </div>
             <script>
                 var input = document

diff --git a/src/js/fileupload2.js b/src/js/fileupload2.js
@@ -82,23 +82,26 @@ $(document).ready(function() {
 
             switch (checksumAlgName) {
                 case 'MD5':
-                    js.src = "https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.0.0/md5.js";
+                    js.src = "https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.2.0/md5.js";
                     break;
                 case 'SHA-1':
-                    js.src = "https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.0.0/sha1.js";
+                    js.src = "https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.2.0/sha1.js";
                     break;
                 case 'SHA-256':
-                    js.src = "https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.0.0/sha256.js";
+                    js.src = "https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.2.0/sha256.js";
                     break;
                 case 'SHA-512':
-                    js.src = "https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.0.0/x64-core.js";
+                    js.src = "https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.2.0/x64-core.js";
+                    //Make async false to avoid sha512 loading before the x64-core which can cause an error
+                    js.async = false;
                     head.appendChild(js);
                     js = document.createElement("script");
                     js.type = "text/javascript";
-                    js.src = "https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.0.0/sha512.js";
+                    js.src = "https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.2.0/sha512.js";
+                    js.async = false;
                     break;
                 default:
-                    js.src = "https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.0.0/md5.js";
+                    js.src = "https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.2.0/md5.js";
             }
             head.appendChild(js);
             retrieveDatasetInfo();
@@ -188,6 +191,7 @@ function initSpanTxt(htmlId, key) {
 function addMessage(type, key) {
     $('#messages').html('').append($('<div/>').addClass(type).text(getLocalizedString(dvLocale, key)));
 }
+
 async function populatePageMetadata(data) {
     var mdFields = data.metadataBlocks.citation.fields;
     var title = "";
@@ -663,14 +667,28 @@ function queueFileForDirectUpload(file) {
     }
     var fUpload = new fileUpload(file);
     let send = true;
-    let path = file.webkitRelativePath.substring(file.webkitRelativePath.indexOf('/') + 1);
-    console.log(path);
+    let origPath = file.webkitRelativePath.substring(file.webkitRelativePath.indexOf('/') + 1);
+
+    //Remove filename part
+    let path =origPath.substring(0, origPath.length - file.name.length);
+    let badPath = (path.match(/^[\w\d_\-\.\\\/ ]*$/)===null);
+    if(badPath) {
+      if($('.warn').length==0) {
+        addMessage('warn', 'msgRequiredPathOrFileNameChange');
+      }
+      //Munge path according to rules
+      path = path.replace(/[^\w\d_\\.\\\/ ]+/g,'_');
+    }
+    //Re-Add filename, munge filename if needed
+    path=path.concat(file.name.replace(/[\/:*?|;#]/g,'_'));
+
+    //Now check versus existing files
     if (path in existingFiles) {
         send = false;
     } else if (removeExtension(path) in convertedFileNameMap) {
         send = false;
     }
-    rawFileMap[path] = file;
+    rawFileMap[origPath] = file;
     let i = rawFileMap.length;
     //startUploads();
     if (send) {
@@ -686,8 +704,18 @@ function queueFileForDirectUpload(file) {
     if (!send) {
         row.addClass('file-exists');
     }
-    row.append($('<input/>').prop('type', 'checkbox').prop('id', 'file_' + fileBlock.children().length).prop('checked', send))
-        .append($('<div/>').addClass('ui-fileupload-filename').text(path))
+    let badChars = !(fUpload.file.name.match(/[[\/:*?|;#]/)===null);
+    if(badChars) {
+      if($('.warn').length==0) {
+          addMessage('warn', 'msgRequiredPathOrFileNameChange');
+      }
+    }
+    row.append($('<input/>').prop('type', 'checkbox').prop('id', 'file_' + fileBlock.children().length).prop('checked', send));
+    let fnameElement = $('<div/>').addClass('ui-fileupload-filename').text(origPath);
+    if(badPath || badChars) {
+      fnameElement.addClass('badchars');
+    }
+    row.append(fnameElement)
         .append($('<div/>').text(file.size)).append($('<div/>').addClass('ui - fileupload - progress'))
         .append($('<div/>').addClass('ui - fileupload - cancel'));
     console.log('adding click handler for file_' + fileBlock.children().length);
@@ -824,10 +852,13 @@ async function directUploadFinished() {
                     console.log(fup.file.webkitRelativePath + ' : ' + fup.storageId);
                     let entry = {};
                     entry.storageIdentifier = fup.storageId;
-                    entry.fileName = fup.file.name;
+                    //Remove bad file name chars
+                    entry.fileName = fup.file.name.replace(/[\/:*?|;#]/g,'_');
                     let path = fup.file.webkitRelativePath;
                     console.log(path);
                     path = path.substring(path.indexOf('/'), path.lastIndexOf('/'));
+                    //Remove bad path chars
+                    path = path.replace(/[^\w\d_\\.\\\/ ]+/g,'_');
                     if (path.length !== 0) {
                         entry.directoryLabel = path;
                     }

diff --git a/src/js/lang.js b/src/js/lang.js
@@ -15,6 +15,7 @@ const translations = {
         msgNoFile: "No files to upload. Check some files, or refresh to start over.",
         msgUploadCompleteRegistering: "Uploads to S3 complete. Now registering all files with the dataset. This may take some time for large numbers of files.",
         msgUploadComplete: "Upload complete, all files in dataset. Close this window and refresh your dataset page to see the uploaded files.",
+        msgRequiredPathOrFileNameChange: "The highlighted path/file(s) below contain one or more disallowed characters (paths can only contain a-Z, 0-9, '_', '-', '.', '\', '/' and ' ', and filenames cannot contain any of '/;:|?*#' ). Disallowed characters will be replaced by an underscore ('_') if the file(s) are uploaded.",
     },
     fr: {
         title: "Envoi d'un dossier",
@@ -31,6 +32,7 @@ const translations = {
         msgNoFile: "Aucun fichier à envoyer. Cochez certains fichiers ou rafraîchissez la page pour recommencer.",
         msgUploadCompleteRegistering: "Envois vers S3 terminés. Enregistrement de tous les fichiers en cours dans le jeu de données. Cela peut prendre du temps pour un grand nombre de fichiers.",
         msgUploadComplete: "Envoi terminé, tous les fichiers sont dans le jeu de données. Fermez cette fenêtre et rafraîchissez la page de votre jeu de données pour voir les fichiers envoyés.",
+        msgRequiredPathOrFileNameChange: "Le(s) chemin(s) en surbrillance ci-dessous contiennent un ou plusieurs caractères non autorisés (les chemins ne peuvent contenir que a-Z, 0-9, '_', '-', '.', '\', '/' et ' ', et les noms de fichiers ne peuvent contenir aucun des éléments '/;:|?*#' ). Les caractères non autorisés seront remplacés par un trait de soulignement (« _ ») si le(s) fichier(s) sont téléchargés.",
     },
 };