add backup helper, messages and channels are now sorted correctly, up…

…date docs

README.md

@@ -13,16 +13,25 @@ View your JSON [DiscordChatExporter](Tyrrrz/DiscordChatExporter) exports as if y
 - View media files locally
 - Browse guild or direct messages
 - Discord Markdown rendering support
+- Command generator to extend your export with more messages (backup helper)
 
 
 
 ## Quick start (Windows)
 Using prebuilt binaries is the easiest way to use this tool on Windows.
 1. Download the latest release from [releases page](https://github.com/slatinsky/DiscordChatExporter-frontend/releases)
 2. Extract the archive
-3. Move your JSON+media [DiscordChatExporter](Tyrrrz/DiscordChatExporter) exports to `/static/input/` folder ([supported exports](#custom_anchor_name)). Folder structure inside this folder doesn't matter, script will find everything it needs.
+3. Move your JSON+media [DiscordChatExporter](https://github.com/Tyrrrz/DiscordChatExporter) exports to `/static/input/` folder ([supported exports](#supported-exports)). Folder structure inside this folder doesn't matter, script will find everything it needs.
 4. Run `START_VIEWER.bat` - DiscordChatExporter-frontend will open in your default browser
 
+## Upgrade guide
+Want to upgrade from previous version? Follow these steps:
+
+1. Download the latest release from [releases page](https://github.com/slatinsky/DiscordChatExporter-frontend/releases)
+2. Extract the archive
+3. Move your `/static/input/` folder to the new release folder. DO NOT MOVE `/static/data/` folder, because the format is not compatible between releases.
+4. Delete old release folder
+
 ## Linux
 This tool uses Sveltekit and Python3 as main dependencies. You won't be able to run premade Windows batch scripts, but running this tool on Linux is possible. Linux support is WIP.
 
@@ -47,6 +56,10 @@ The main requirement is that media files (`--media True --reuse-media True`) are
 ## How to view threads
 - This viewer supports viewing threads, but they need to be exported by [DiscordChatExporter](https://github.com/Tyrrrz/DiscordChatExporter). Export them the same way you export channels (`--channel`), but instead of CHANNEL_ID, use THREAD_ID. Because threads are channels.
 
+Don't know how to get THREAD_IDs? Handy backup helper is included to extend your backup and to find missing threads. You can find it at the end of channel list.
+
+![](docs/backup_helper.png)
+
 # Development
 You don't need to follow development steps if you don't need to modify the code.
 
@@ -176,13 +189,13 @@ But should work on any Windows 10 / Windows 11 x64 computer.
 
 
 ## Roadmap / planned features:
-- rerun preprocess only if it is needed
 - Better handling of edge cases (if something is missing in the backup)
 - Better GUI
 - make readme easy to understand
 - Linux support (docker?)
 - Improve code readability
-- Discord forums support
+- online mode - view media files directly from Discord servers
+- Discord forums support - waiting for DiscordChatExporter export support
 
 ## Why this tool was made
 [DiscordChatExporter](https://github.com/Tyrrrz/DiscordChatExporter) is a well made tool to export Discord chats. But to actually view them, you had to download them in HTML format, which more inconvenient to parse than JSON. And If you wanted to extend your backup, it would be broken into multiple files, which is not very convenient.

preprocess/preprocess.py

@@ -7,6 +7,10 @@
 import shutil
 from hashlib import sha256
 
+def pad_id(id):
+    return str(id).zfill(24)
+    # return str(id).rjust(24, '0')
+
 
 class GuildPreprocess:
     def __init__(self, guild_id, input_dir, json_filepaths, media_filepaths):
@@ -15,12 +19,52 @@ def __init__(self, guild_id, input_dir, json_filepaths, media_filepaths):
         self.json_filepaths = json_filepaths
         self.media_filepaths = media_filepaths
 
+    ## if any field in data has key 'id', pad it with zeros for fast sorting
+    def pad_ids(self, data):
+        data['guild']['id'] = pad_id(data['guild']['id'])
+        data['channel']['id'] = pad_id(data['channel']['id'])
+
+        data['channel']['categoryId'] = pad_id(data['channel']['categoryId'])
+
+
+        # for message in data.messages:
+        for message in data['messages']:
+            message['id'] = pad_id(message['id'])
+            message['author']['id'] = pad_id(message['author']['id'])
+
+            for reaction in message['reactions']:
+                reaction['emoji']['id'] = pad_id(reaction['emoji']['id'])
+
+            # mentions
+            for mention in message['mentions']:
+                mention['id'] = pad_id(mention['id'])
+
+            # attachments
+            for attachment in message['attachments']:
+                attachment['id'] = pad_id(attachment['id'])
+
+            # sticker
+            for sticker in message['stickers']:
+                sticker['id'] = pad_id(sticker['id'])
+
+            if 'reference' in message:
+                if message['reference']['messageId'] is not None:
+                    message['reference']['messageId'] = pad_id(message['reference']['messageId'])
+                if message['reference']['channelId'] is not None:
+                    message['reference']['channelId'] = pad_id(message['reference']['channelId'])
+                if message['reference']['guildId'] is not None:
+                    message['reference']['guildId'] = pad_id(message['reference']['guildId'])
+        return data
+
+
     def read_channels_messages_from_files(self):
         channels = {}
         messages = {}
         for path in self.json_filepaths:
             with open(path, 'r', encoding="utf8") as f:
+                print("Reading file: " + path)
                 data = json.load(f)
+                data = self.pad_ids(data)
                 channel = data['channel']
 
                 if channel['id'] not in channels:
@@ -300,6 +344,12 @@ def process(self):
         # step 1 - read data from json files
         channels, messages = self.read_channels_messages_from_files()
 
+        # sort messages dict by key
+        messages = dict(sorted(messages.items()))
+
+        # sort channels dict by key
+        channels = dict(sorted(channels.items()))
+
         # print message count
         print("Message count: " + str(len(messages)))
 
@@ -452,7 +502,8 @@ def process(self):
                     continue
                 if 'guild' not in data:  # this is not a channel export, but a downloaded media json file
                     continue
-                guild_id = data['guild']['id']
+                guild_id = pad_id(data['guild']['id'])
+                data['guild']['id'] = guild_id
                 guilds[guild_id] = data['guild']
                 if guild_id not in json_paths_by_guild:
                     json_paths_by_guild[guild_id] = []

src/routes/channels/[guildId]/+layout.svelte

@@ -63,6 +63,9 @@
 				</div>
 			{/each}
 		{/each}
+		{#if data.guildId != '0'}
+			<a href="/channels/{data.guildId}/continue">Backup helper</a>
+		{/if}
 	</div>
 	<div id="header">
 		{#key data.channelId}

src/routes/channels/[guildId]/[channelId]/Message.svelte

@@ -93,8 +93,11 @@
 	function addReferencedMessage() {
 		if (message.reference) {
 			// console.log(message.reference);
-			message.referencedMessage =
-				guild.messages[message.reference.channelId][message.reference.messageId];
+			try {
+				message.referencedMessage = guild.messages[message.reference.channelId][message.reference.messageId];
+			} catch (e) {
+				console.warn("Couldn't find referenced message");  // if channel is not exported and we try to get first referenced message from thread
+			}
 			if (message.referencedMessage && message.referencedMessage?.authorId) {
 				message.referencedMessage.author = guild.authors[message.referencedMessage.authorId];
 				// delete message.referencedMessage.authorId;

src/routes/channels/[guildId]/[channelId]/WatchHash.svelte

@@ -56,7 +56,7 @@
 			console.error('recursion depth exceeded');
 			return;
 		}
-		let elMessage = document.getElementById(messageId);
+		let elMessage = document.getElementById(messageId.toString().padStart(24, '0'));
 		if (elMessage) {
 			elMessage.scrollIntoView();
 			console.log('found message', messageId, "- recursion depth", recursionDepth);
@@ -87,7 +87,7 @@
 		if (bestRange) {
 			let first = bestRange[0];
 			let last = bestRange[1];
-			let el = document.querySelector('.message-group[data-mgfirst="' + first + '"][data-mglast="' + last + '"]')
+			let el = document.querySelector('.message-group[data-mgfirst="' + first.toString().padStart(24, '0') + '"][data-mglast="' + last.toString().padStart(24, '0') + '"]')
 				if (el) {
 					el.scrollIntoView();
 					setTimeout(() => {
@@ -127,6 +127,7 @@
 			try {
 				searchForMessageId(BigInt(messageId));
 			} catch (e) {
+				console.error(e);
 				console.warn("Url hash does not contain a valid message id");
 			}
 		}