diff --git a/README.md b/README.md index 50b4b455..f8ce56de 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ View your JSON [DiscordChatExporter](Tyrrrz/DiscordChatExporter) exports as if y - View media files locally - Browse guild or direct messages - Discord Markdown rendering support +- Command generator to extend your export with more messages (backup helper) @@ -20,9 +21,17 @@ View your JSON [DiscordChatExporter](Tyrrrz/DiscordChatExporter) exports as if y Using prebuilt binaries is the easiest way to use this tool on Windows. 1. Download the latest release from [releases page](https://github.com/slatinsky/DiscordChatExporter-frontend/releases) 2. Extract the archive -3. Move your JSON+media [DiscordChatExporter](Tyrrrz/DiscordChatExporter) exports to `/static/input/` folder ([supported exports](#custom_anchor_name)). Folder structure inside this folder doesn't matter, script will find everything it needs. +3. Move your JSON+media [DiscordChatExporter](https://github.com/Tyrrrz/DiscordChatExporter) exports to `/static/input/` folder ([supported exports](#supported-exports)). Folder structure inside this folder doesn't matter, script will find everything it needs. 4. Run `START_VIEWER.bat` - DiscordChatExporter-frontend will open in your default browser +## Upgrade guide +Want to upgrade from previous version? Follow these steps: + +1. Download the latest release from [releases page](https://github.com/slatinsky/DiscordChatExporter-frontend/releases) +2. Extract the archive +3. Move your `/static/input/` folder to the new release folder. DO NOT MOVE `/static/data/` folder, because the format is not compatible between releases. +4. Delete old release folder + ## Linux This tool uses Sveltekit and Python3 as main dependencies. You won't be able to run premade Windows batch scripts, but running this tool on Linux is possible. Linux support is WIP. @@ -47,6 +56,10 @@ The main requirement is that media files (`--media True --reuse-media True`) are ## How to view threads - This viewer supports viewing threads, but they need to be exported by [DiscordChatExporter](https://github.com/Tyrrrz/DiscordChatExporter). Export them the same way you export channels (`--channel`), but instead of CHANNEL_ID, use THREAD_ID. Because threads are channels. +Don't know how to get THREAD_IDs? Handy backup helper is included to extend your backup and to find missing threads. You can find it at the end of channel list. + +![](docs/backup_helper.png) + # Development You don't need to follow development steps if you don't need to modify the code. @@ -176,13 +189,13 @@ But should work on any Windows 10 / Windows 11 x64 computer. ## Roadmap / planned features: -- rerun preprocess only if it is needed - Better handling of edge cases (if something is missing in the backup) - Better GUI - make readme easy to understand - Linux support (docker?) - Improve code readability -- Discord forums support +- online mode - view media files directly from Discord servers +- Discord forums support - waiting for DiscordChatExporter export support ## Why this tool was made [DiscordChatExporter](https://github.com/Tyrrrz/DiscordChatExporter) is a well made tool to export Discord chats. But to actually view them, you had to download them in HTML format, which more inconvenient to parse than JSON. And If you wanted to extend your backup, it would be broken into multiple files, which is not very convenient. diff --git a/docs/backup_helper.png b/docs/backup_helper.png new file mode 100644 index 00000000..5c0d12c0 Binary files /dev/null and b/docs/backup_helper.png differ diff --git a/preprocess/preprocess.py b/preprocess/preprocess.py index 4dd436da..5dd51b98 100644 --- a/preprocess/preprocess.py +++ b/preprocess/preprocess.py @@ -7,6 +7,10 @@ import shutil from hashlib import sha256 +def pad_id(id): + return str(id).zfill(24) + # return str(id).rjust(24, '0') + class GuildPreprocess: def __init__(self, guild_id, input_dir, json_filepaths, media_filepaths): @@ -15,12 +19,52 @@ def __init__(self, guild_id, input_dir, json_filepaths, media_filepaths): self.json_filepaths = json_filepaths self.media_filepaths = media_filepaths + ## if any field in data has key 'id', pad it with zeros for fast sorting + def pad_ids(self, data): + data['guild']['id'] = pad_id(data['guild']['id']) + data['channel']['id'] = pad_id(data['channel']['id']) + + data['channel']['categoryId'] = pad_id(data['channel']['categoryId']) + + + # for message in data.messages: + for message in data['messages']: + message['id'] = pad_id(message['id']) + message['author']['id'] = pad_id(message['author']['id']) + + for reaction in message['reactions']: + reaction['emoji']['id'] = pad_id(reaction['emoji']['id']) + + # mentions + for mention in message['mentions']: + mention['id'] = pad_id(mention['id']) + + # attachments + for attachment in message['attachments']: + attachment['id'] = pad_id(attachment['id']) + + # sticker + for sticker in message['stickers']: + sticker['id'] = pad_id(sticker['id']) + + if 'reference' in message: + if message['reference']['messageId'] is not None: + message['reference']['messageId'] = pad_id(message['reference']['messageId']) + if message['reference']['channelId'] is not None: + message['reference']['channelId'] = pad_id(message['reference']['channelId']) + if message['reference']['guildId'] is not None: + message['reference']['guildId'] = pad_id(message['reference']['guildId']) + return data + + def read_channels_messages_from_files(self): channels = {} messages = {} for path in self.json_filepaths: with open(path, 'r', encoding="utf8") as f: + print("Reading file: " + path) data = json.load(f) + data = self.pad_ids(data) channel = data['channel'] if channel['id'] not in channels: @@ -300,6 +344,12 @@ def process(self): # step 1 - read data from json files channels, messages = self.read_channels_messages_from_files() + # sort messages dict by key + messages = dict(sorted(messages.items())) + + # sort channels dict by key + channels = dict(sorted(channels.items())) + # print message count print("Message count: " + str(len(messages))) @@ -452,7 +502,8 @@ def process(self): continue if 'guild' not in data: # this is not a channel export, but a downloaded media json file continue - guild_id = data['guild']['id'] + guild_id = pad_id(data['guild']['id']) + data['guild']['id'] = guild_id guilds[guild_id] = data['guild'] if guild_id not in json_paths_by_guild: json_paths_by_guild[guild_id] = [] diff --git a/src/routes/channels/[guildId]/+layout.svelte b/src/routes/channels/[guildId]/+layout.svelte index 35c946bb..07d21d7a 100644 --- a/src/routes/channels/[guildId]/+layout.svelte +++ b/src/routes/channels/[guildId]/+layout.svelte @@ -63,6 +63,9 @@ {/each} {/each} + {#if data.guildId != '0'} + Backup helper + {/if}
This tool will generate commands to extend your server backup without downloading duplicate messages. Just run these commands, move your new backup to '/static/input' and run START_VIEWER.bat again to apply changes. This tool WON'T FIND NEW CHANNELS
+ +These commands will extend your export of existing channels:
+ {#key commandsUpdateLength} + + {/key} + +These commands will extend your export of existing threads:
+ {#key commandsUpdateTLength} + + {/key} + +These commands will download missing threads. These threads were found in the backup, but weren't exported:
+ {#key commandsThreadsLength} + {#if newThreads.length > 0} + + {:else} +Great! All threads are archived
+ {/if} + {/key} +