From 1f3ee3c91058e62aade6fbb509d98a702ce6ae45 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 18:51:04 +0000 Subject: [PATCH 01/10] Bump vite from 5.4.0 to 5.4.6 in /assets Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 5.4.0 to 5.4.6. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v5.4.6/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v5.4.6/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- assets/package-lock.json | 34 +++++++++++++++++----------------- assets/package.json | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/assets/package-lock.json b/assets/package-lock.json index 8ba664e3d..4ccada05b 100644 --- a/assets/package-lock.json +++ b/assets/package-lock.json @@ -15,7 +15,7 @@ "sass": "^1.75.0", "typescript": "^5.4", "typescript-eslint": "8.0.0-alpha.39", - "vite": "^5.2" + "vite": "^5.4" }, "devDependencies": { "@testing-library/dom": "^10.1.0", @@ -4051,9 +4051,9 @@ } }, "node_modules/picocolors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", - "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.0.tgz", + "integrity": "sha512-TQ92mBOW0l3LeMeyLV6mzy/kWr8lkd/hp3mTg7wYK7zJhuBStmGMBG0BdeDZS/dZx1IukaX6Bk11zcln25o1Aw==" }, "node_modules/picomatch": { "version": "2.3.1", @@ -4078,9 +4078,9 @@ } }, "node_modules/postcss": { - "version": "8.4.41", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.41.tgz", - "integrity": "sha512-TesUflQ0WKZqAvg52PWL6kHgLKP6xB6heTOdoYM0Wt2UHyxNa4K25EZZMgKns3BH1RLVbZCREPpLY0rhnNoHVQ==", + "version": "8.4.47", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.47.tgz", + "integrity": "sha512-56rxCq7G/XfB4EkXq9Egn5GCqugWvDFjafDOThIdMBsI15iqPqR5r15TfSr1YPYeEI19YeaXMCbY6u88Y76GLQ==", "funding": [ { "type": "opencollective", @@ -4097,8 +4097,8 @@ ], "dependencies": { "nanoid": "^3.3.7", - "picocolors": "^1.0.1", - "source-map-js": "^1.2.0" + "picocolors": "^1.1.0", + "source-map-js": "^1.2.1" }, "engines": { "node": "^10 || ^12 || >=14" @@ -4425,9 +4425,9 @@ } }, "node_modules/source-map-js": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", - "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", "engines": { "node": ">=0.10.0" } @@ -4784,13 +4784,13 @@ } }, "node_modules/vite": { - "version": "5.4.0", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.0.tgz", - "integrity": "sha512-5xokfMX0PIiwCMCMb9ZJcMyh5wbBun0zUzKib+L65vAZ8GY9ePZMXxFrHbr/Kyll2+LSCY7xtERPpxkBDKngwg==", + "version": "5.4.6", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.6.tgz", + "integrity": "sha512-IeL5f8OO5nylsgzd9tq4qD2QqI0k2CQLGrWD0rCN0EQJZpBK5vJAx0I+GDkMOXxQX/OfFHMuLIx6ddAxGX/k+Q==", "dependencies": { "esbuild": "^0.21.3", - "postcss": "^8.4.40", - "rollup": "^4.13.0" + "postcss": "^8.4.43", + "rollup": "^4.20.0" }, "bin": { "vite": "bin/vite.js" diff --git a/assets/package.json b/assets/package.json index 9da504397..ad514620e 100644 --- a/assets/package.json +++ b/assets/package.json @@ -20,7 +20,7 @@ "sass": "^1.75.0", "typescript": "^5.4", "typescript-eslint": "8.0.0-alpha.39", - "vite": "^5.2" + "vite": "^5.4" }, "devDependencies": { "@testing-library/dom": "^10.1.0", From 09c93f3215dd060151a71048734c344d2ee3027b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 03:51:25 +0000 Subject: [PATCH 02/10] Bump rollup from 4.20.0 to 4.22.4 in /assets Bumps [rollup](https://github.com/rollup/rollup) from 4.20.0 to 4.22.4. - [Release notes](https://github.com/rollup/rollup/releases) - [Changelog](https://github.com/rollup/rollup/blob/master/CHANGELOG.md) - [Commits](https://github.com/rollup/rollup/compare/v4.20.0...v4.22.4) --- updated-dependencies: - dependency-name: rollup dependency-type: indirect ... Signed-off-by: dependabot[bot] --- assets/package-lock.json | 134 +++++++++++++++++++-------------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/assets/package-lock.json b/assets/package-lock.json index 4ccada05b..300a9f34a 100644 --- a/assets/package-lock.json +++ b/assets/package-lock.json @@ -825,9 +825,9 @@ } }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.20.0.tgz", - "integrity": "sha512-TSpWzflCc4VGAUJZlPpgAJE1+V60MePDQnBd7PPkpuEmOy8i87aL6tinFGKBFKuEDikYpig72QzdT3QPYIi+oA==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.22.4.tgz", + "integrity": "sha512-Fxamp4aEZnfPOcGA8KSNEohV8hX7zVHOemC8jVBoBUHu5zpJK/Eu3uJwt6BMgy9fkvzxDaurgj96F/NiLukF2w==", "cpu": [ "arm" ], @@ -837,9 +837,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.20.0.tgz", - "integrity": "sha512-u00Ro/nok7oGzVuh/FMYfNoGqxU5CPWz1mxV85S2w9LxHR8OoMQBuSk+3BKVIDYgkpeOET5yXkx90OYFc+ytpQ==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.22.4.tgz", + "integrity": "sha512-VXoK5UMrgECLYaMuGuVTOx5kcuap1Jm8g/M83RnCHBKOqvPPmROFJGQaZhGccnsFtfXQ3XYa4/jMCJvZnbJBdA==", "cpu": [ "arm64" ], @@ -849,9 +849,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.20.0.tgz", - "integrity": "sha512-uFVfvzvsdGtlSLuL0ZlvPJvl6ZmrH4CBwLGEFPe7hUmf7htGAN+aXo43R/V6LATyxlKVC/m6UsLb7jbG+LG39Q==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.22.4.tgz", + "integrity": "sha512-xMM9ORBqu81jyMKCDP+SZDhnX2QEVQzTcC6G18KlTQEzWK8r/oNZtKuZaCcHhnsa6fEeOBionoyl5JsAbE/36Q==", "cpu": [ "arm64" ], @@ -861,9 +861,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.20.0.tgz", - "integrity": "sha512-xbrMDdlev53vNXexEa6l0LffojxhqDTBeL+VUxuuIXys4x6xyvbKq5XqTXBCEUA8ty8iEJblHvFaWRJTk/icAQ==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.22.4.tgz", + "integrity": "sha512-aJJyYKQwbHuhTUrjWjxEvGnNNBCnmpHDvrb8JFDbeSH3m2XdHcxDd3jthAzvmoI8w/kSjd2y0udT+4okADsZIw==", "cpu": [ "x64" ], @@ -873,9 +873,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.20.0.tgz", - "integrity": "sha512-jMYvxZwGmoHFBTbr12Xc6wOdc2xA5tF5F2q6t7Rcfab68TT0n+r7dgawD4qhPEvasDsVpQi+MgDzj2faOLsZjA==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.22.4.tgz", + "integrity": "sha512-j63YtCIRAzbO+gC2L9dWXRh5BFetsv0j0va0Wi9epXDgU/XUi5dJKo4USTttVyK7fGw2nPWK0PbAvyliz50SCQ==", "cpu": [ "arm" ], @@ -885,9 +885,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.20.0.tgz", - "integrity": "sha512-1asSTl4HKuIHIB1GcdFHNNZhxAYEdqML/MW4QmPS4G0ivbEcBr1JKlFLKsIRqjSwOBkdItn3/ZDlyvZ/N6KPlw==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.22.4.tgz", + "integrity": "sha512-dJnWUgwWBX1YBRsuKKMOlXCzh2Wu1mlHzv20TpqEsfdZLb3WoJW2kIEsGwLkroYf24IrPAvOT/ZQ2OYMV6vlrg==", "cpu": [ "arm" ], @@ -897,9 +897,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.20.0.tgz", - "integrity": "sha512-COBb8Bkx56KldOYJfMf6wKeYJrtJ9vEgBRAOkfw6Ens0tnmzPqvlpjZiLgkhg6cA3DGzCmLmmd319pmHvKWWlQ==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.22.4.tgz", + "integrity": "sha512-AdPRoNi3NKVLolCN/Sp4F4N1d98c4SBnHMKoLuiG6RXgoZ4sllseuGioszumnPGmPM2O7qaAX/IJdeDU8f26Aw==", "cpu": [ "arm64" ], @@ -909,9 +909,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.20.0.tgz", - "integrity": "sha512-+it+mBSyMslVQa8wSPvBx53fYuZK/oLTu5RJoXogjk6x7Q7sz1GNRsXWjn6SwyJm8E/oMjNVwPhmNdIjwP135Q==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.22.4.tgz", + "integrity": "sha512-Gl0AxBtDg8uoAn5CCqQDMqAx22Wx22pjDOjBdmG0VIWX3qUBHzYmOKh8KXHL4UpogfJ14G4wk16EQogF+v8hmA==", "cpu": [ "arm64" ], @@ -921,9 +921,9 @@ ] }, "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.20.0.tgz", - "integrity": "sha512-yAMvqhPfGKsAxHN8I4+jE0CpLWD8cv4z7CK7BMmhjDuz606Q2tFKkWRY8bHR9JQXYcoLfopo5TTqzxgPUjUMfw==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.22.4.tgz", + "integrity": "sha512-3aVCK9xfWW1oGQpTsYJJPF6bfpWfhbRnhdlyhak2ZiyFLDaayz0EP5j9V1RVLAAxlmWKTDfS9wyRyY3hvhPoOg==", "cpu": [ "ppc64" ], @@ -933,9 +933,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.20.0.tgz", - "integrity": "sha512-qmuxFpfmi/2SUkAw95TtNq/w/I7Gpjurx609OOOV7U4vhvUhBcftcmXwl3rqAek+ADBwSjIC4IVNLiszoj3dPA==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.22.4.tgz", + "integrity": "sha512-ePYIir6VYnhgv2C5Xe9u+ico4t8sZWXschR6fMgoPUK31yQu7hTEJb7bCqivHECwIClJfKgE7zYsh1qTP3WHUA==", "cpu": [ "riscv64" ], @@ -945,9 +945,9 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.20.0.tgz", - "integrity": "sha512-I0BtGXddHSHjV1mqTNkgUZLnS3WtsqebAXv11D5BZE/gfw5KoyXSAXVqyJximQXNvNzUo4GKlCK/dIwXlz+jlg==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.22.4.tgz", + "integrity": "sha512-GqFJ9wLlbB9daxhVlrTe61vJtEY99/xB3C8e4ULVsVfflcpmR6c8UZXjtkMA6FhNONhj2eA5Tk9uAVw5orEs4Q==", "cpu": [ "s390x" ], @@ -957,9 +957,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.20.0.tgz", - "integrity": "sha512-y+eoL2I3iphUg9tN9GB6ku1FA8kOfmF4oUEWhztDJ4KXJy1agk/9+pejOuZkNFhRwHAOxMsBPLbXPd6mJiCwew==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.22.4.tgz", + "integrity": "sha512-87v0ol2sH9GE3cLQLNEy0K/R0pz1nvg76o8M5nhMR0+Q+BBGLnb35P0fVz4CQxHYXaAOhE8HhlkaZfsdUOlHwg==", "cpu": [ "x64" ], @@ -969,9 +969,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.20.0.tgz", - "integrity": "sha512-hM3nhW40kBNYUkZb/r9k2FKK+/MnKglX7UYd4ZUy5DJs8/sMsIbqWK2piZtVGE3kcXVNj3B2IrUYROJMMCikNg==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.22.4.tgz", + "integrity": "sha512-UV6FZMUgePDZrFjrNGIWzDo/vABebuXBhJEqrHxrGiU6HikPy0Z3LfdtciIttEUQfuDdCn8fqh7wiFJjCNwO+g==", "cpu": [ "x64" ], @@ -981,9 +981,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.20.0.tgz", - "integrity": "sha512-psegMvP+Ik/Bg7QRJbv8w8PAytPA7Uo8fpFjXyCRHWm6Nt42L+JtoqH8eDQ5hRP7/XW2UiIriy1Z46jf0Oa1kA==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.22.4.tgz", + "integrity": "sha512-BjI+NVVEGAXjGWYHz/vv0pBqfGoUH0IGZ0cICTn7kB9PyjrATSkX+8WkguNjWoj2qSr1im/+tTGRaY+4/PdcQw==", "cpu": [ "arm64" ], @@ -993,9 +993,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.20.0.tgz", - "integrity": "sha512-GabekH3w4lgAJpVxkk7hUzUf2hICSQO0a/BLFA11/RMxQT92MabKAqyubzDZmMOC/hcJNlc+rrypzNzYl4Dx7A==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.22.4.tgz", + "integrity": "sha512-SiWG/1TuUdPvYmzmYnmd3IEifzR61Tragkbx9D3+R8mzQqDBz8v+BvZNDlkiTtI9T15KYZhP0ehn3Dld4n9J5g==", "cpu": [ "ia32" ], @@ -1005,9 +1005,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.20.0.tgz", - "integrity": "sha512-aJ1EJSuTdGnM6qbVC4B5DSmozPTqIag9fSzXRNNo+humQLG89XpPgdt16Ia56ORD7s+H8Pmyx44uczDQ0yDzpg==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.22.4.tgz", + "integrity": "sha512-j8pPKp53/lq9lMXN57S8cFz0MynJk8OWNuUnXct/9KCpKU7DgU3bYMJhwWmcqC0UU29p8Lr0/7KEVcaM6bf47Q==", "cpu": [ "x64" ], @@ -4266,9 +4266,9 @@ } }, "node_modules/rollup": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.20.0.tgz", - "integrity": "sha512-6rbWBChcnSGzIlXeIdNIZTopKYad8ZG8ajhl78lGRLsI2rX8IkaotQhVas2Ma+GPxJav19wrSzvRvuiv0YKzWw==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.22.4.tgz", + "integrity": "sha512-vD8HJ5raRcWOyymsR6Z3o6+RzfEPCnVLMFJ6vRslO1jt4LO6dUo5Qnpg7y4RkZFM2DMe3WUirkI5c16onjrc6A==", "dependencies": { "@types/estree": "1.0.5" }, @@ -4280,22 +4280,22 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.20.0", - "@rollup/rollup-android-arm64": "4.20.0", - "@rollup/rollup-darwin-arm64": "4.20.0", - "@rollup/rollup-darwin-x64": "4.20.0", - "@rollup/rollup-linux-arm-gnueabihf": "4.20.0", - "@rollup/rollup-linux-arm-musleabihf": "4.20.0", - "@rollup/rollup-linux-arm64-gnu": "4.20.0", - "@rollup/rollup-linux-arm64-musl": "4.20.0", - "@rollup/rollup-linux-powerpc64le-gnu": "4.20.0", - "@rollup/rollup-linux-riscv64-gnu": "4.20.0", - "@rollup/rollup-linux-s390x-gnu": "4.20.0", - "@rollup/rollup-linux-x64-gnu": "4.20.0", - "@rollup/rollup-linux-x64-musl": "4.20.0", - "@rollup/rollup-win32-arm64-msvc": "4.20.0", - "@rollup/rollup-win32-ia32-msvc": "4.20.0", - "@rollup/rollup-win32-x64-msvc": "4.20.0", + "@rollup/rollup-android-arm-eabi": "4.22.4", + "@rollup/rollup-android-arm64": "4.22.4", + "@rollup/rollup-darwin-arm64": "4.22.4", + "@rollup/rollup-darwin-x64": "4.22.4", + "@rollup/rollup-linux-arm-gnueabihf": "4.22.4", + "@rollup/rollup-linux-arm-musleabihf": "4.22.4", + "@rollup/rollup-linux-arm64-gnu": "4.22.4", + "@rollup/rollup-linux-arm64-musl": "4.22.4", + "@rollup/rollup-linux-powerpc64le-gnu": "4.22.4", + "@rollup/rollup-linux-riscv64-gnu": "4.22.4", + "@rollup/rollup-linux-s390x-gnu": "4.22.4", + "@rollup/rollup-linux-x64-gnu": "4.22.4", + "@rollup/rollup-linux-x64-musl": "4.22.4", + "@rollup/rollup-win32-arm64-msvc": "4.22.4", + "@rollup/rollup-win32-ia32-msvc": "4.22.4", + "@rollup/rollup-win32-x64-msvc": "4.22.4", "fsevents": "~2.3.2" } }, From 34c9f76330de993d1d966797950dd25897f7e9ae Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 27 Sep 2024 19:53:06 -0400 Subject: [PATCH 03/10] Fix tag short description indexing --- lib/philomena/tags/search_index.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/philomena/tags/search_index.ex b/lib/philomena/tags/search_index.ex index ec681a3f7..4589c0656 100644 --- a/lib/philomena/tags/search_index.ex +++ b/lib/philomena/tags/search_index.ex @@ -71,7 +71,7 @@ defmodule Philomena.Tags.SearchIndex do category: tag.category, aliased: !!tag.aliased_tag, description: tag.description, - short_description: tag.description + short_description: tag.short_description } end end From a08510f21efa7ec3a32082f70d4420b64b73c156 Mon Sep 17 00:00:00 2001 From: Yay295 Date: Mon, 28 Oct 2024 15:13:42 -0500 Subject: [PATCH 04/10] Add Bluesky scraper (#290) * add bluesky scraper * use com.atproto.sync.getBlob to get original bluesky image * ignore data after bluesky post id * fix json access and missing function * fix bluesky fullsize image regex --- lib/philomena_proxy/scrapers.ex | 1 + lib/philomena_proxy/scrapers/bluesky.ex | 48 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 lib/philomena_proxy/scrapers/bluesky.ex diff --git a/lib/philomena_proxy/scrapers.ex b/lib/philomena_proxy/scrapers.ex index a96f08176..08674d442 100644 --- a/lib/philomena_proxy/scrapers.ex +++ b/lib/philomena_proxy/scrapers.ex @@ -21,6 +21,7 @@ defmodule PhilomenaProxy.Scrapers do } @scrapers [ + PhilomenaProxy.Scrapers.Bluesky, PhilomenaProxy.Scrapers.Deviantart, PhilomenaProxy.Scrapers.Pillowfort, PhilomenaProxy.Scrapers.Twitter, diff --git a/lib/philomena_proxy/scrapers/bluesky.ex b/lib/philomena_proxy/scrapers/bluesky.ex new file mode 100644 index 000000000..598d14706 --- /dev/null +++ b/lib/philomena_proxy/scrapers/bluesky.ex @@ -0,0 +1,48 @@ +defmodule PhilomenaProxy.Scrapers.Bluesky do + @moduledoc false + + alias PhilomenaProxy.Scrapers.Scraper + alias PhilomenaProxy.Scrapers + + @behaviour Scraper + + @url_regex ~r|https://bsky\.app/profile/([^/]+)/post/([^/?#]+)| + @fullsize_image_regex ~r|.*/img/feed_fullsize/plain/([^/]+)/([^@]+).*| + @blob_image_url_pattern "https://bsky.social/xrpc/com.atproto.sync.getBlob/?did=\\1&cid=\\2" + + @spec can_handle?(URI.t(), String.t()) :: boolean() + def can_handle?(_uri, url) do + String.match?(url, @url_regex) + end + + @spec scrape(URI.t(), Scrapers.url()) :: Scrapers.scrape_result() + def scrape(_uri, url) do + [handle, id] = Regex.run(@url_regex, url, capture: :all_but_first) + + api_url_resolve_handle = + "https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=#{handle}" + + did = PhilomenaProxy.Http.get(api_url_resolve_handle) |> json!() |> Map.fetch!(:did) + + api_url_get_posts = + "https://public.api.bsky.app/xrpc/app.bsky.feed.getPosts?uris=at://#{did}/app.bsky.feed.post/#{id}" + + post_json = PhilomenaProxy.Http.get(api_url_get_posts) |> json!() |> Map.fetch!(:posts) |> hd + + %{ + source_url: url, + author_name: post_json["author"]["handle"], + description: post_json["record"]["text"], + images: + post_json["embed"]["images"] + |> Enum.map( + &%{ + url: String.replace(&1["fullsize"], @fullsize_image_regex, @blob_image_url_pattern), + camo_url: PhilomenaProxy.Camo.image_url(&1["thumb"]) + } + ) + } + end + + defp json!({:ok, %{body: body, status: 200}}), do: Jason.decode!(body) +end From c2cb6a1649a22ab4657f35f4117b0df0efb14f8e Mon Sep 17 00:00:00 2001 From: Liam Date: Thu, 7 Nov 2024 13:46:16 -0500 Subject: [PATCH 05/10] Change PhilomenaQuery.Batch to Stream-based instead of callback-based --- lib/mix/tasks/upload_to_s3.ex | 8 +- lib/philomena/user_downvote_wipe.ex | 9 +- .../workers/tag_change_revert_worker.ex | 4 +- lib/philomena_query/batch.ex | 89 ++++++++++--------- lib/philomena_query/search.ex | 6 +- 5 files changed, 67 insertions(+), 49 deletions(-) diff --git a/lib/mix/tasks/upload_to_s3.ex b/lib/mix/tasks/upload_to_s3.ex index 87ab48d3a..61c61f87f 100644 --- a/lib/mix/tasks/upload_to_s3.ex +++ b/lib/mix/tasks/upload_to_s3.ex @@ -121,7 +121,9 @@ defmodule Mix.Tasks.UploadToS3 do end defp upload_typical(queryable, batch_size, file_root, new_file_root, field_name) do - Batch.record_batches(queryable, [batch_size: batch_size], fn models -> + queryable + |> Batch.record_batches(batch_size: batch_size) + |> Enum.each(fn models -> models |> Task.async_stream(&upload_typical_model(&1, file_root, new_file_root, field_name), timeout: :infinity @@ -142,7 +144,9 @@ defmodule Mix.Tasks.UploadToS3 do end defp upload_images(queryable, batch_size, file_root, new_file_root) do - Batch.record_batches(queryable, [batch_size: batch_size], fn models -> + queryable + |> Batch.record_batches(batch_size: batch_size) + |> Enum.each(fn models -> models |> Task.async_stream(&upload_image_model(&1, file_root, new_file_root), timeout: :infinity) |> Stream.run() diff --git a/lib/philomena/user_downvote_wipe.ex b/lib/philomena/user_downvote_wipe.ex index 6fb217fe9..b958ab9ba 100644 --- a/lib/philomena/user_downvote_wipe.ex +++ b/lib/philomena/user_downvote_wipe.ex @@ -15,7 +15,8 @@ defmodule Philomena.UserDownvoteWipe do ImageVote |> where(user_id: ^user.id, up: false) - |> Batch.query_batches([id_field: :image_id], fn queryable -> + |> Batch.query_batches(id_field: :image_id) + |> Enum.each(fn queryable -> {_, image_ids} = Repo.delete_all(select(queryable, [i_v], i_v.image_id)) {count, nil} = @@ -31,7 +32,8 @@ defmodule Philomena.UserDownvoteWipe do if upvotes_and_faves_too do ImageVote |> where(user_id: ^user.id, up: true) - |> Batch.query_batches([id_field: :image_id], fn queryable -> + |> Batch.query_batches(id_field: :image_id) + |> Enum.each(fn queryable -> {_, image_ids} = Repo.delete_all(select(queryable, [i_v], i_v.image_id)) {count, nil} = @@ -46,7 +48,8 @@ defmodule Philomena.UserDownvoteWipe do ImageFave |> where(user_id: ^user.id) - |> Batch.query_batches([id_field: :image_id], fn queryable -> + |> Batch.query_batches(id_field: :image_id) + |> Enum.each(fn queryable -> {_, image_ids} = Repo.delete_all(select(queryable, [i_f], i_f.image_id)) {count, nil} = diff --git a/lib/philomena/workers/tag_change_revert_worker.ex b/lib/philomena/workers/tag_change_revert_worker.ex index 519b8404a..80058a31f 100644 --- a/lib/philomena/workers/tag_change_revert_worker.ex +++ b/lib/philomena/workers/tag_change_revert_worker.ex @@ -27,7 +27,9 @@ defmodule Philomena.TagChangeRevertWorker do batch_size = attributes["batch_size"] || 100 attributes = Map.delete(attributes, "batch_size") - Batch.query_batches(queryable, [batch_size: batch_size], fn queryable -> + queryable + |> Batch.query_batches(batch_size: batch_size) + |> Enum.each(fn queryable -> ids = Repo.all(select(queryable, [tc], tc.id)) TagChanges.mass_revert(ids, cast_ip(atomify_keys(attributes))) end) diff --git a/lib/philomena_query/batch.ex b/lib/philomena_query/batch.ex index 918a3b5e9..20bdc3648 100644 --- a/lib/philomena_query/batch.ex +++ b/lib/philomena_query/batch.ex @@ -25,24 +25,32 @@ defmodule PhilomenaQuery.Batch do @type id_field :: {:id_field, atom()} @type batch_options :: [batch_size() | id_field()] - @typedoc """ - The callback for `record_batches/3`. + @doc """ + Stream schema structures on a queryable, using batches to avoid locking. - Takes a list of schema structs which were returned in the batch. Return value is ignored. - """ - @type record_batch_callback :: ([struct()] -> any()) + Valid options: + * `batch_size` (integer) - the number of records to load per batch + * `id_field` (atom) - the name of the field containing the ID - @typedoc """ - The callback for `query_batches/3`. + ## Example + + queryable = from i in Image, where: i.image_width >= 1920 + + queryable + |> PhilomenaQuery.Batch.record_batches() + |> Enum.each(fn image -> IO.inspect(image.id) end) - Takes an `m:Ecto.Query` that can be processed with `m:Philomena.Repo` query commands, such - as `Philomena.Repo.update_all/3` or `Philomena.Repo.delete_all/2`. Return value is ignored. """ - @type query_batch_callback :: ([Ecto.Query.t()] -> any()) + @spec records(queryable(), batch_options()) :: Enumerable.t() + def records(queryable, opts \\ []) do + queryable + |> query_batches(opts) + |> Stream.flat_map(&Repo.all/1) + end @doc """ - Execute a callback with lists of schema structures on a queryable, - using batches to avoid locking. + Stream lists of schema structures on a queryable, using batches to avoid + locking. Valid options: * `batch_size` (integer) - the number of records to load per batch @@ -56,16 +64,20 @@ defmodule PhilomenaQuery.Batch do Enum.each(images, &IO.inspect(&1.id)) end - PhilomenaQuery.Batch.record_batches(queryable, cb) + queryable + |> PhilomenaQuery.Batch.record_batches() + |> Enum.each(cb) """ - @spec record_batches(queryable(), batch_options(), record_batch_callback()) :: [] - def record_batches(queryable, opts \\ [], callback) do - query_batches(queryable, opts, &callback.(Repo.all(&1))) + @spec record_batches(queryable(), batch_options()) :: Enumerable.t() + def record_batches(queryable, opts \\ []) do + queryable + |> query_batches(opts) + |> Stream.map(&Repo.all/1) end @doc """ - Execute a callback with bulk queries on a queryable, using batches to avoid locking. + Stream bulk queries on a queryable, using batches to avoid locking. Valid options: * `batch_size` (integer) - the number of records to load per batch @@ -76,41 +88,36 @@ defmodule PhilomenaQuery.Batch do > If you are looking to receive schema structures (e.g., you are querying for `Image`s, > and you want to receive `Image` objects, then use `record_batches/3` instead. - An `m:Ecto.Query` which selects all IDs in the current batch is passed into the callback - during each invocation. + `m:Ecto.Query` structs which select the IDs in each batch are streamed out. ## Example queryable = from ui in ImageVote, where: ui.user_id == 1234 - opts = [id_field: :image_id] - - cb = fn bulk_query -> - Repo.delete_all(bulk_query) - end - - PhilomenaQuery.Batch.query_batches(queryable, opts, cb) + queryable + |> PhilomenaQuery.Batch.query_batches(id_field: :image_id) + |> Enum.each(fn batch_query -> Repo.delete_all(batch_query) end) """ - @spec query_batches(queryable(), batch_options(), query_batch_callback()) :: [] - def query_batches(queryable, opts \\ [], callback) do - ids = load_ids(queryable, -1, opts) - - query_batches(queryable, opts, callback, ids) - end - - defp query_batches(_queryable, _opts, _callback, []), do: [] - - defp query_batches(queryable, opts, callback, ids) do + @spec query_batches(queryable(), batch_options()) :: Enumerable.t(Ecto.Query.t()) + def query_batches(queryable, opts \\ []) do id_field = Keyword.get(opts, :id_field, :id) - queryable - |> where([m], field(m, ^id_field) in ^ids) - |> callback.() + Stream.unfold( + load_ids(queryable, -1, opts), + fn + [] -> + # Stop when no more results are produced + nil - ids = load_ids(queryable, Enum.max(ids), opts) + ids -> + # Process results and output next query + output = where(queryable, [m], field(m, ^id_field) in ^ids) + next_ids = load_ids(queryable, Enum.max(ids), opts) - query_batches(queryable, opts, callback, ids) + {output, next_ids} + end + ) end defp load_ids(queryable, max_id, opts) do diff --git a/lib/philomena_query/search.ex b/lib/philomena_query/search.ex index 2519e5804..cd02137c0 100644 --- a/lib/philomena_query/search.ex +++ b/lib/philomena_query/search.ex @@ -199,11 +199,13 @@ defmodule PhilomenaQuery.Search do Search.reindex(query, Image, batch_size: 5000) """ - @spec reindex(queryable(), schema_module(), Batch.batch_options()) :: [] + @spec reindex(queryable(), schema_module(), Batch.batch_options()) :: :ok def reindex(queryable, module, opts \\ []) do index = @policy.index_for(module) - Batch.record_batches(queryable, opts, fn records -> + queryable + |> Batch.record_batches(opts) + |> Enum.each(fn records -> lines = Enum.flat_map(records, fn record -> doc = index.as_json(record) From 7312c647a810b84d1359622b2d8666f159b280f5 Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 8 Nov 2024 23:26:51 -0500 Subject: [PATCH 06/10] Add additional linting workflows --- .github/workflows/elixir.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index a92aa72fe..734c0b682 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -31,6 +31,9 @@ jobs: - run: docker compose pull - run: docker compose build + - name: mix format + run: docker compose run app mix format --check-formatted + - name: Build and test run: docker compose run app run-test @@ -50,6 +53,21 @@ jobs: - uses: actions/checkout@v4 - uses: crate-ci/typos@master + cargo: + name: Rust Linting and Unit Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + + - name: cargo fmt + run: (cd native/philomena && cargo fmt --check) + + - name: cargo clippy + run: (cd native/philomena && cargo clippy -- -D warnings) + lint-and-test: name: 'JavaScript Linting and Unit Tests' runs-on: ubuntu-latest @@ -80,4 +98,4 @@ jobs: working-directory: ./assets - run: npm run build - working-directory: ./assets \ No newline at end of file + working-directory: ./assets From 5fab189f82379df3d92c49660ef53c42a2e21c59 Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 8 Nov 2024 23:27:46 -0500 Subject: [PATCH 07/10] Run cargo fmt --- native/philomena/src/camo.rs | 4 ++-- native/philomena/src/markdown.rs | 25 +++++++++++++++++-------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/native/philomena/src/camo.rs b/native/philomena/src/camo.rs index 5a72b3f68..df47a2e86 100644 --- a/native/philomena/src/camo.rs +++ b/native/philomena/src/camo.rs @@ -1,8 +1,8 @@ +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine; use ring::hmac; use std::env; use url::Url; -use base64::Engine; -use base64::engine::general_purpose::URL_SAFE_NO_PAD; fn trusted_host(mut url: Url) -> Option { url.set_port(Some(443)).ok()?; diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs index af9762336..c92e6a0fb 100644 --- a/native/philomena/src/markdown.rs +++ b/native/philomena/src/markdown.rs @@ -1,5 +1,5 @@ -use comrak::ComrakOptions; use crate::camo; +use comrak::ComrakOptions; use rustler::{MapIterator, Term}; use std::collections::HashMap; use std::env; @@ -19,19 +19,28 @@ fn common_options() -> ComrakOptions { options.extension.camoifier = Some(|s| camo::image_url(s).unwrap_or_else(|| String::from(""))); if let Ok(domains) = env::var("SITE_DOMAINS") { - options.extension.philomena_domains = Some(domains.split(',').map(|s| s.to_string()).collect::>()); + options.extension.philomena_domains = Some( + domains + .split(',') + .map(|s| s.to_string()) + .collect::>(), + ); } options } fn map_to_hashmap(map: Term) -> Option> { - Some(MapIterator::new(map)?.map(|(key, value)| { - let key: String = key.decode().unwrap_or_else(|_| String::from("")); - let value: String = value.decode().unwrap_or_else(|_| String::from("")); - - (key, value) - }).collect()) + Some( + MapIterator::new(map)? + .map(|(key, value)| { + let key: String = key.decode().unwrap_or_else(|_| String::from("")); + let value: String = value.decode().unwrap_or_else(|_| String::from("")); + + (key, value) + }) + .collect(), + ) } pub fn to_html(input: String, reps: Term) -> String { From 9c797e8d643c645b975ccd76a1bb0c7b96faf4fe Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 8 Nov 2024 23:55:50 -0500 Subject: [PATCH 08/10] Rust code refactor --- lib/philomena_web/markdown_renderer.ex | 6 ++-- native/philomena/Cargo.lock | 2 ++ native/philomena/Cargo.toml | 6 ++-- native/philomena/src/camo.rs | 20 ++++++----- native/philomena/src/domains.rs | 14 ++++++++ native/philomena/src/lib.rs | 11 +++--- native/philomena/src/markdown.rs | 49 +++++++------------------- 7 files changed, 52 insertions(+), 56 deletions(-) create mode 100644 native/philomena/src/domains.rs diff --git a/lib/philomena_web/markdown_renderer.ex b/lib/philomena_web/markdown_renderer.ex index 7caff5c90..0b1080085 100644 --- a/lib/philomena_web/markdown_renderer.ex +++ b/lib/philomena_web/markdown_renderer.ex @@ -75,8 +75,7 @@ defmodule PhilomenaWeb.MarkdownRenderer do defp render_representations(images, conn) do loaded_images = load_images(images) - images - |> Enum.map(fn group -> + Map.new(images, fn group -> img = loaded_images[Enum.at(group, 0)] text = "#{Enum.at(group, 0)}#{Enum.at(group, 1)}" @@ -131,8 +130,7 @@ defmodule PhilomenaWeb.MarkdownRenderer do |> Phoenix.HTML.Safe.to_iodata() |> IO.iodata_to_binary() - [text, string_contents] + {text, string_contents} end) - |> Map.new(fn [id, html] -> {id, html} end) end end diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index 0ca747199..495a6815b 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -260,7 +260,9 @@ version = "0.3.0" dependencies = [ "base64", "comrak", + "http", "jemallocator", + "regex", "ring", "rustler", "url", diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index 942d39bc6..20cb3b9bf 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -10,11 +10,13 @@ path = "src/lib.rs" crate-type = ["dylib"] [dependencies] +base64 = "0.21" comrak = { git = "https://github.com/philomena-dev/comrak", branch = "main", default-features = false } +http = "0.2" jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] } -rustler = "0.28" +regex = "1" ring = "0.16" -base64 = "0.21" +rustler = "0.28" url = "2.3" [profile.release] diff --git a/native/philomena/src/camo.rs b/native/philomena/src/camo.rs index df47a2e86..c79f12e1f 100644 --- a/native/philomena/src/camo.rs +++ b/native/philomena/src/camo.rs @@ -11,7 +11,7 @@ fn trusted_host(mut url: Url) -> Option { Some(url.to_string()) } -fn untrusted_host(url: Url, camo_host: String, camo_key: String) -> Option { +fn untrusted_host(url: Url, camo_host: &str, camo_key: &str) -> Option { let camo_url = format!("https://{}", camo_host); let key = hmac::Key::new(hmac::HMAC_SHA1_FOR_LEGACY_USE_ONLY, camo_key.as_ref()); let tag = hmac::sign(&key, url.to_string().as_bytes()); @@ -27,20 +27,24 @@ fn untrusted_host(url: Url, camo_host: String, camo_key: String) -> Option Option { +pub fn image_url(uri: &str) -> Option { let cdn_host = env::var("CDN_HOST").ok()?; - let camo_host = env::var("CAMO_HOST").unwrap_or_else(|_| String::from("")); - let camo_key = env::var("CAMO_KEY").unwrap_or_else(|_| String::from("")); + let camo_host = env::var("CAMO_HOST").unwrap_or_else(|_| "".into()); + let camo_key = env::var("CAMO_KEY").unwrap_or_else(|_| "".into()); if camo_key.is_empty() { - return Some(uri); + return Some(uri.into()); } - let url = Url::parse(&uri).ok()?; + let url = Url::parse(uri).ok()?; match url.host_str() { Some(hostname) if hostname == cdn_host || hostname == camo_host => trusted_host(url), - Some(_) => untrusted_host(url, camo_host, camo_key), - None => Some(String::from("")), + Some(_) => untrusted_host(url, &camo_host, &camo_key), + None => Some("".into()), } } + +pub fn image_url_careful(uri: &str) -> String { + image_url(uri).unwrap_or_else(|| "".into()) +} diff --git a/native/philomena/src/domains.rs b/native/philomena/src/domains.rs new file mode 100644 index 000000000..c90ddb6f4 --- /dev/null +++ b/native/philomena/src/domains.rs @@ -0,0 +1,14 @@ +use std::env; + +pub fn get() -> Option> { + if let Ok(domains) = env::var("SITE_DOMAINS") { + return Some( + domains + .split(',') + .map(|s| s.to_string()) + .collect::>(), + ); + } + + None +} diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index f5317aa30..bcbadcb6f 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -1,7 +1,8 @@ use jemallocator::Jemalloc; -use rustler::Term; +use std::collections::HashMap; mod camo; +mod domains; mod markdown; #[global_allocator] @@ -15,18 +16,18 @@ rustler::init! { // Markdown NIF wrappers. #[rustler::nif(schedule = "DirtyCpu")] -fn markdown_to_html(input: String, reps: Term) -> String { +fn markdown_to_html(input: &str, reps: HashMap) -> String { markdown::to_html(input, reps) } #[rustler::nif(schedule = "DirtyCpu")] -fn markdown_to_html_unsafe(input: String, reps: Term) -> String { +fn markdown_to_html_unsafe(input: &str, reps: HashMap) -> String { markdown::to_html_unsafe(input, reps) } // Camo NIF wrappers. #[rustler::nif] -fn camo_image_url(input: String) -> String { - camo::image_url(input).unwrap_or_else(|| String::from("")) +fn camo_image_url(input: &str) -> String { + camo::image_url_careful(input) } diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs index c92e6a0fb..a927a6e50 100644 --- a/native/philomena/src/markdown.rs +++ b/native/philomena/src/markdown.rs @@ -1,11 +1,9 @@ -use crate::camo; -use comrak::ComrakOptions; -use rustler::{MapIterator, Term}; +use crate::{camo, domains}; +use comrak::Options; use std::collections::HashMap; -use std::env; -fn common_options() -> ComrakOptions { - let mut options = ComrakOptions::default(); +fn common_options() -> Options { + let mut options = Options::default(); options.extension.autolink = true; options.extension.table = true; options.extension.description_lists = true; @@ -16,47 +14,24 @@ fn common_options() -> ComrakOptions { options.render.hardbreaks = true; options.render.github_pre_lang = true; - options.extension.camoifier = Some(|s| camo::image_url(s).unwrap_or_else(|| String::from(""))); - - if let Ok(domains) = env::var("SITE_DOMAINS") { - options.extension.philomena_domains = Some( - domains - .split(',') - .map(|s| s.to_string()) - .collect::>(), - ); - } + options.extension.camoifier = Some(|s| camo::image_url_careful(&s)); + options.extension.philomena_domains = domains::get(); options } -fn map_to_hashmap(map: Term) -> Option> { - Some( - MapIterator::new(map)? - .map(|(key, value)| { - let key: String = key.decode().unwrap_or_else(|_| String::from("")); - let value: String = value.decode().unwrap_or_else(|_| String::from("")); - - (key, value) - }) - .collect(), - ) -} - -pub fn to_html(input: String, reps: Term) -> String { +pub fn to_html(input: &str, reps: HashMap) -> String { let mut options = common_options(); options.render.escape = true; + options.extension.philomena_replacements = Some(reps); - options.extension.philomena_replacements = map_to_hashmap(reps); - - comrak::markdown_to_html(&input, &options) + comrak::markdown_to_html(input, &options) } -pub fn to_html_unsafe(input: String, reps: Term) -> String { +pub fn to_html_unsafe(input: &str, reps: HashMap) -> String { let mut options = common_options(); options.render.unsafe_ = true; + options.extension.philomena_replacements = Some(reps); - options.extension.philomena_replacements = map_to_hashmap(reps); - - comrak::markdown_to_html(&input, &options) + comrak::markdown_to_html(input, &options) } From 1de4a7b82011ec73a38d939710e0e1750a424000 Mon Sep 17 00:00:00 2001 From: Liam Date: Sun, 3 Nov 2024 19:17:01 -0500 Subject: [PATCH 09/10] Add framework for zip data export --- lib/philomena/data_exports/aggregator.ex | 171 ++++++++++++++++++++ lib/philomena/data_exports/zip_generator.ex | 56 +++++++ lib/philomena/native.ex | 12 ++ lib/philomena_query/batch.ex | 3 +- native/philomena/Cargo.lock | 169 ++++++++++++++++++- native/philomena/Cargo.toml | 1 + native/philomena/src/lib.rs | 36 ++++- native/philomena/src/zip.rs | 69 ++++++++ 8 files changed, 509 insertions(+), 8 deletions(-) create mode 100644 lib/philomena/data_exports/aggregator.ex create mode 100644 lib/philomena/data_exports/zip_generator.ex create mode 100644 native/philomena/src/zip.rs diff --git a/lib/philomena/data_exports/aggregator.ex b/lib/philomena/data_exports/aggregator.ex new file mode 100644 index 000000000..5dcb82b3d --- /dev/null +++ b/lib/philomena/data_exports/aggregator.ex @@ -0,0 +1,171 @@ +defmodule Philomena.DataExports.Aggregator do + @moduledoc """ + Data generation module for data export logic. + """ + + import Ecto.Query + alias PhilomenaQuery.Batch + + # Direct PII + alias Philomena.Donations.Donation + alias Philomena.UserFingerprints.UserFingerprint + alias Philomena.UserIps.UserIp + alias Philomena.UserNameChanges.UserNameChange + alias Philomena.Users.User + + # UGC for export + alias Philomena.ArtistLinks.ArtistLink + alias Philomena.Badges.Award + alias Philomena.Comments.Comment + alias Philomena.Commissions.Commission + alias Philomena.DnpEntries.DnpEntry + alias Philomena.DuplicateReports.DuplicateReport + alias Philomena.Filters.Filter + alias Philomena.ImageFaves.ImageFave + alias Philomena.ImageHides.ImageHide + alias Philomena.ImageVotes.ImageVote + alias Philomena.Images.Image + alias Philomena.PollVotes.PollVote + alias Philomena.Posts.Post + alias Philomena.Reports.Report + alias Philomena.SourceChanges.SourceChange + alias Philomena.TagChanges.TagChange + alias Philomena.Topics.Topic + alias Philomena.Bans.User, as: UserBan + + # Direct UGC from form submission + @user_columns [ + :created_at, + :name, + :email, + :description, + :current_filter_id, + :spoiler_type, + :theme, + :images_per_page, + :show_large_thumbnails, + :show_sidebar_and_watched_images, + :fancy_tag_field_on_upload, + :fancy_tag_field_on_edit, + :fancy_tag_field_in_settings, + :autorefresh_by_default, + :anonymous_by_default, + :comments_newest_first, + :comments_always_jump_to_last, + :comments_per_page, + :watch_on_reply, + :watch_on_new_topic, + :watch_on_upload, + :messages_newest_first, + :serve_webm, + :no_spoilered_in_watched, + :watched_images_query_str, + :watched_images_exclude_str, + :use_centered_layout, + :personal_title, + :hide_vote_counts, + :scale_large_images + ] + + # All these also have created_at and are selected by user_id + @indirect_columns [ + {Donation, [:email, :amount, :fee, :note]}, + {UserFingerprint, [:fingerprint, :uses, :updated_at]}, + {UserIp, [:ip, :uses, :updated_at]}, + {UserNameChange, [:name]}, + {ArtistLink, [:aasm_state, :uri, :public, :tag_id]}, + {Award, [:label, :badge_name, :badge_id]}, + {Comment, + [ + :ip, + :fingerprint, + :user_agent, + :referrer, + :anonymous, + :image_id, + :edited_at, + :edit_reason, + :body + ]}, + {Commission, + [:open, :sheet_image_id, :categories, :information, :contact, :will_create, :will_not_create]}, + {DnpEntry, [:tag_id, :aasm_state, :dnp_type, :hide_reason, :feedback, :reason, :instructions], + :requesting_user_id}, + {DuplicateReport, [:reason, :image_id, :duplicate_of_image_id]}, + {Filter, + [ + :name, + :description, + :public, + :hidden_complex_str, + :spoilered_complex_str, + :hidden_tag_ids, + :spoilered_tag_ids + ]}, + {ImageFave, [:image_id], :user_id, :image_id}, + {ImageHide, [:image_id], :user_id, :image_id}, + {ImageVote, [:image_id, :up], :user_id, :image_id}, + {Image, [:ip, :fingerprint, :user_agent, :referrer, :anonymous, :description]}, + {PollVote, [:rank, :poll_option_id]}, + {Post, + [:ip, :fingerprint, :user_agent, :referrer, :anonymous, :edited_at, :edit_reason, :body]}, + {Report, + [:ip, :fingerprint, :user_agent, :referrer, :reason, :reportable_id, :reportable_type]}, + {SourceChange, [:ip, :fingerprint, :user_agent, :referrer, :image_id, :added, :value]}, + {TagChange, + [:ip, :fingerprint, :user_agent, :referrer, :image_id, :added, :tag_id, :tag_name_cache]}, + {Topic, [:title, :anonymous, :forum_id]}, + {UserBan, [:reason, :generated_ban_id]} + ] + + @doc """ + Get all of the export data for the given user. + """ + def get_for_user(user_id) do + [select_user(user_id)] ++ select_indirect(user_id) + end + + defp select_user(user_id) do + select_schema_by_key(user_id, User, @user_columns, :id) + end + + defp select_indirect(user_id) do + Enum.map(@indirect_columns, fn + {schema_name, columns} -> + select_schema_by_key(user_id, schema_name, columns) + + {schema_name, columns, key_column} -> + select_schema_by_key(user_id, schema_name, columns, key_column) + + {schema_name, columns, key_column, id_field} -> + select_schema_by_key(user_id, schema_name, columns, key_column, id_field) + end) + end + + defp select_schema_by_key( + user_id, + schema_name, + columns, + key_column \\ :user_id, + id_field \\ :id + ) do + table_name = schema_name.__schema__(:source) + columns = [:created_at] ++ columns + + {"#{table_name}.jsonl", + schema_name + |> where([s], field(s, ^key_column) == ^user_id) + |> select([s], map(s, ^columns)) + |> Batch.records(id_field: id_field) + |> results_as_json_lines()} + end + + defp results_as_json_lines(list_of_maps) do + Stream.map(list_of_maps, fn map -> + map + |> Map.new(fn {k, v} -> {k, to_string(v)} end) + |> Jason.encode!() + |> Kernel.<>("\n") + end) + end +end diff --git a/lib/philomena/data_exports/zip_generator.ex b/lib/philomena/data_exports/zip_generator.ex new file mode 100644 index 000000000..8c5aaf7eb --- /dev/null +++ b/lib/philomena/data_exports/zip_generator.ex @@ -0,0 +1,56 @@ +defmodule Philomena.DataExports.ZipGenerator do + @moduledoc """ + ZIP file generator for an export. + """ + + alias Philomena.Native + + @doc """ + Write the ZIP file for the given aggregate data. + + Expects a list of 2-tuples, with the first element being the name of the + file to generate, and the second element being a stream which generates the + binary contents of the file. + """ + @spec generate(Path.t(), Enumerable.t()) :: :ok | atom() + def generate(filename, aggregate) do + case Native.zip_open_writer(filename) do + {:ok, zip} -> + stream_aggregate(zip, aggregate) + + error -> + error + end + end + + @spec stream_aggregate(reference(), Enumerable.t()) :: {:ok, reference()} | :error + defp stream_aggregate(zip, aggregate) do + aggregate + |> Enum.reduce_while(:ok, fn {name, content_stream}, _ -> + with :ok <- Native.zip_start_file(zip, name), + :ok <- stream_file_data(zip, content_stream) do + {:cont, :ok} + else + error -> + {:halt, error} + end + end) + |> case do + :ok -> + Native.zip_finish(zip) + + error -> + error + end + end + + @spec stream_file_data(reference(), Enumerable.t(iodata())) :: :ok | :error + defp stream_file_data(zip, content_stream) do + Enum.reduce_while(content_stream, :ok, fn iodata, _ -> + case Native.zip_write(zip, IO.iodata_to_binary(iodata)) do + :ok -> {:cont, :ok} + error -> {:halt, error} + end + end) + end +end diff --git a/lib/philomena/native.ex b/lib/philomena/native.ex index a67dc33f4..14eeaa17f 100644 --- a/lib/philomena/native.ex +++ b/lib/philomena/native.ex @@ -11,4 +11,16 @@ defmodule Philomena.Native do @spec camo_image_url(String.t()) :: String.t() def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded) + + @spec zip_open_writer(Path.t()) :: {:ok, reference()} | {:error, atom()} + def zip_open_writer(_path), do: :erlang.nif_error(:nif_not_loaded) + + @spec zip_start_file(reference(), String.t()) :: :ok | :error + def zip_start_file(_zip, _name), do: :erlang.nif_error(:nif_not_loaded) + + @spec zip_write(reference(), binary()) :: :ok | :error + def zip_write(_zip, _data), do: :erlang.nif_error(:nif_not_loaded) + + @spec zip_finish(reference()) :: :ok | :error + def zip_finish(_zip), do: :erlang.nif_error(:nif_not_loaded) end diff --git a/lib/philomena_query/batch.ex b/lib/philomena_query/batch.ex index 20bdc3648..bb9c2c74d 100644 --- a/lib/philomena_query/batch.ex +++ b/lib/philomena_query/batch.ex @@ -125,8 +125,9 @@ defmodule PhilomenaQuery.Batch do batch_size = Keyword.get(opts, :batch_size, 1000) queryable - |> exclude(:preload) |> exclude(:order_by) + |> exclude(:preload) + |> exclude(:select) |> order_by(asc: ^id_field) |> where([m], field(m, ^id_field) > ^max_id) |> select([m], field(m, ^id_field)) diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index 495a6815b..623227a12 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -11,6 +17,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "base64" version = "0.21.7" @@ -57,6 +72,21 @@ dependencies = [ "unicode_categories", ] +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "darling" version = "0.20.9" @@ -92,6 +122,17 @@ dependencies = [ "syn", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "derive_builder" version = "0.20.0" @@ -129,12 +170,39 @@ version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "322ef0094744e63628e6f0eb2295517f79276a5b342a4c2ff3042566ca181d4e" +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "entities" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "flate2" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -150,6 +218,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "hashbrown" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" + [[package]] name = "heck" version = "0.4.1" @@ -183,6 +257,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "itoa" version = "1.0.11" @@ -230,6 +314,12 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.21" @@ -238,9 +328,18 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "miniz_oxide" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] [[package]] name = "once_cell" @@ -266,13 +365,14 @@ dependencies = [ "ring", "rustler", "url", + "zip", ] [[package]] name = "proc-macro2" -version = "1.0.80" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -363,6 +463,12 @@ dependencies = [ "unreachable", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "slug" version = "0.1.5" @@ -387,15 +493,35 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.59" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "thiserror" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -561,3 +687,34 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "zip" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index 20cb3b9bf..9ef7caaca 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -18,6 +18,7 @@ regex = "1" ring = "0.16" rustler = "0.28" url = "2.3" +zip = { version = "2.2.0", features = ["deflate"], default-features = false } [profile.release] opt-level = 3 diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index bcbadcb6f..6c8f96f79 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -1,16 +1,28 @@ use jemallocator::Jemalloc; +use rustler::{Atom, Binary, Env, Term}; use std::collections::HashMap; mod camo; mod domains; mod markdown; +mod zip; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; rustler::init! { "Elixir.Philomena.Native", - [markdown_to_html, markdown_to_html_unsafe, camo_image_url] + [ + markdown_to_html, markdown_to_html_unsafe, camo_image_url, + zip_open_writer, zip_start_file, zip_write, zip_finish + ], + load = load +} + +// Setup. + +fn load(env: Env, arg: Term) -> bool { + zip::load(env, arg) } // Markdown NIF wrappers. @@ -31,3 +43,25 @@ fn markdown_to_html_unsafe(input: &str, reps: HashMap) -> String fn camo_image_url(input: &str) -> String { camo::image_url_careful(input) } + +// Zip NIF wrappers. + +#[rustler::nif] +fn zip_open_writer(path: &str) -> Result { + zip::open_writer(path) +} + +#[rustler::nif] +fn zip_start_file(writer: zip::WriterResourceArc, name: &str) -> Atom { + zip::start_file(writer, name) +} + +#[rustler::nif(schedule = "DirtyCpu")] +fn zip_write(writer: zip::WriterResourceArc, data: Binary) -> Atom { + zip::write(writer, data.as_slice()) +} + +#[rustler::nif(schedule = "DirtyCpu")] +fn zip_finish(writer: zip::WriterResourceArc) -> Atom { + zip::finish(writer) +} diff --git a/native/philomena/src/zip.rs b/native/philomena/src/zip.rs new file mode 100644 index 000000000..60ef15be5 --- /dev/null +++ b/native/philomena/src/zip.rs @@ -0,0 +1,69 @@ +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::sync::Mutex; + +use rustler::{Atom, Env, ResourceArc, Term}; +use zip::{write::SimpleFileOptions, CompressionMethod, ZipWriter}; + +mod atoms { + rustler::atoms! { + ok, + error, + } +} + +pub struct WriterResource { + inner: Mutex>>, +} + +pub type WriterResourceArc = ResourceArc; + +pub fn load(env: Env, _: Term) -> bool { + rustler::resource!(WriterResource, env); + true +} + +fn with_writer(writer: WriterResourceArc, f: F) -> Atom +where + F: FnOnce(&mut Option>) -> Option, +{ + let mut guard = match (*writer).inner.lock() { + Ok(g) => g, + Err(_) => return atoms::error(), + }; + + match f(&mut guard) { + Some(_) => atoms::ok(), + None => atoms::error(), + } +} + +pub fn open_writer(path: &str) -> Result { + match OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(path) + { + Ok(file) => Ok(ResourceArc::new(WriterResource { + inner: Mutex::new(Some(ZipWriter::new(file))), + })), + Err(_) => Err(atoms::error()), + } +} + +pub fn start_file(writer: WriterResourceArc, name: &str) -> Atom { + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated); + + with_writer(writer, move |writer| { + writer.as_mut()?.start_file(name, options).ok() + }) +} + +pub fn write(writer: WriterResourceArc, data: &[u8]) -> Atom { + with_writer(writer, move |writer| writer.as_mut()?.write(data).ok()) +} + +pub fn finish(writer: WriterResourceArc) -> Atom { + with_writer(writer, move |writer| writer.take().map(|w| w.finish().ok())) +} From a21189ae10634823d3ed91f3044bfacde14d3b32 Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 9 Nov 2024 00:11:08 -0500 Subject: [PATCH 10/10] Update to Philomena comrak v0.29.0 --- .github/workflows/elixir.yml | 3 + native/philomena/Cargo.lock | 90 ++++++----- native/philomena/Cargo.toml | 2 +- native/philomena/src/domains.rs | 20 +++ native/philomena/src/lib.rs | 2 + native/philomena/src/markdown.rs | 31 +++- native/philomena/src/tests.rs | 257 +++++++++++++++++++++++++++++++ 7 files changed, 356 insertions(+), 49 deletions(-) create mode 100644 native/philomena/src/tests.rs diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index 734c0b682..38b201d43 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -68,6 +68,9 @@ jobs: - name: cargo clippy run: (cd native/philomena && cargo clippy -- -D warnings) + - name: cargo test + run: (cd native/philomena && cargo test) + lint-and-test: name: 'JavaScript Linting and Unit Tests' runs-on: ubuntu-latest diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index 623227a12..bc4af0b4a 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -32,6 +32,29 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "bon" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97493a391b4b18ee918675fb8663e53646fd09321c58b46afa04e8ce2499c869" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2af3eac944c12cdf4423eab70d310da0a8e5851a18ffb192c0a5e3f7ae1663" +dependencies = [ + "darling", + "ident_case", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -44,6 +67,16 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "caseless" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808dab3318747be122cb31d36de18d4d1c81277a76f8332a02b81a3d73463d7f" +dependencies = [ + "regex", + "unicode-normalization", +] + [[package]] name = "cc" version = "1.0.94" @@ -58,12 +91,12 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "comrak" -version = "0.24.1" -source = "git+https://github.com/philomena-dev/comrak?branch=main#6a03dabfc80033b24070dc5826c9225686e3a98a" +version = "0.29.0" +source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.0#0c6fb51a55dddfc1835ed2bedfe3bcb20fb9627e" dependencies = [ - "derive_builder", + "bon", + "caseless", "entities", - "http", "memchr", "once_cell", "regex", @@ -89,9 +122,9 @@ checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "darling" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ "darling_core", "darling_macro", @@ -99,9 +132,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" dependencies = [ "fnv", "ident_case", @@ -113,9 +146,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", @@ -133,37 +166,6 @@ dependencies = [ "syn", ] -[[package]] -name = "derive_builder" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" -dependencies = [ - "derive_builder_core", - "syn", -] - [[package]] name = "deunicode" version = "1.4.4" @@ -463,6 +465,12 @@ dependencies = [ "unreachable", ] +[[package]] +name = "rustversion" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" + [[package]] name = "simd-adler32" version = "0.3.7" diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index 9ef7caaca..19d683428 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -11,7 +11,7 @@ crate-type = ["dylib"] [dependencies] base64 = "0.21" -comrak = { git = "https://github.com/philomena-dev/comrak", branch = "main", default-features = false } +comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.0", default-features = false } http = "0.2" jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] } regex = "1" diff --git a/native/philomena/src/domains.rs b/native/philomena/src/domains.rs index c90ddb6f4..c5626c12b 100644 --- a/native/philomena/src/domains.rs +++ b/native/philomena/src/domains.rs @@ -1,3 +1,5 @@ +use http::Uri; +use regex::Regex; use std::env; pub fn get() -> Option> { @@ -12,3 +14,21 @@ pub fn get() -> Option> { None } + +pub fn relativize(domains: &[String], url: &str) -> Option { + let uri = url.parse::().ok()?; + + if let Some(a) = uri.authority() { + if domains.contains(&a.host().to_string()) { + if let Ok(re) = Regex::new(&format!(r#"^http(s)?://({})"#, regex::escape(a.host()))) { + return Some(re.replace(url, "").into()); + } + } + } + + Some(url.into()) +} + +pub fn relativize_careful(domains: &[String], url: &str) -> String { + relativize(domains, url).unwrap_or_else(|| url.into()) +} diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index 6c8f96f79..ccca12a04 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -5,6 +5,8 @@ use std::collections::HashMap; mod camo; mod domains; mod markdown; +#[cfg(test)] +mod tests; mod zip; #[global_allocator] diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs index a927a6e50..778deb956 100644 --- a/native/philomena/src/markdown.rs +++ b/native/philomena/src/markdown.rs @@ -1,37 +1,54 @@ use crate::{camo, domains}; use comrak::Options; use std::collections::HashMap; +use std::sync::Arc; -fn common_options() -> Options { +pub fn common_options() -> Options { let mut options = Options::default(); + + // Upstream options options.extension.autolink = true; options.extension.table = true; options.extension.description_lists = true; options.extension.superscript = true; options.extension.strikethrough = true; - options.extension.philomena = true; options.parse.smart = true; options.render.hardbreaks = true; options.render.github_pre_lang = true; + options.render.escape = true; - options.extension.camoifier = Some(|s| camo::image_url_careful(&s)); - options.extension.philomena_domains = domains::get(); + // Philomena options + options.extension.underline = true; + options.extension.spoiler = true; + options.extension.greentext = true; + options.extension.subscript = true; + options.extension.philomena = true; + options.render.ignore_empty_links = true; + options.render.ignore_setext = true; + + options.extension.image_url_rewriter = Some(Arc::new(|url: &str| camo::image_url_careful(url))); + + if let Some(domains) = domains::get() { + options.extension.link_url_rewriter = Some(Arc::new(move |url: &str| { + domains::relativize_careful(&domains, url) + })); + } options } pub fn to_html(input: &str, reps: HashMap) -> String { let mut options = common_options(); - options.render.escape = true; - options.extension.philomena_replacements = Some(reps); + options.extension.replacements = Some(reps); comrak::markdown_to_html(input, &options) } pub fn to_html_unsafe(input: &str, reps: HashMap) -> String { let mut options = common_options(); + options.render.escape = false; options.render.unsafe_ = true; - options.extension.philomena_replacements = Some(reps); + options.extension.replacements = Some(reps); comrak::markdown_to_html(input, &options) } diff --git a/native/philomena/src/tests.rs b/native/philomena/src/tests.rs new file mode 100644 index 000000000..9f1f963f5 --- /dev/null +++ b/native/philomena/src/tests.rs @@ -0,0 +1,257 @@ +use std::{collections::HashMap, sync::Arc}; + +use crate::{domains, markdown::*}; + +fn test_options() -> comrak::Options { + let mut options = common_options(); + options.extension.image_url_rewriter = None; + options.extension.link_url_rewriter = None; + options +} + +fn html(input: &str, expected: &str) { + html_opts_w(input, expected, &test_options()); +} + +fn html_opts_i(input: &str, expected: &str, opts: F) +where + F: Fn(&mut comrak::Options), +{ + let mut options = test_options(); + opts(&mut options); + + html_opts_w(input, expected, &options); +} + +fn html_opts_w(input: &str, expected: &str, options: &comrak::Options) { + let output = comrak::markdown_to_html(input, options); + + if output != expected { + println!("Input:"); + println!("========================"); + println!("{}", input); + println!("========================"); + println!("Expected:"); + println!("========================"); + println!("{}", expected); + println!("========================"); + println!("Output:"); + println!("========================"); + println!("{}", output); + println!("========================"); + } + assert_eq!(output, expected); +} + +#[test] +fn subscript() { + html("H%2%O\n", "
H2O
\n"); +} + +#[test] +fn spoiler() { + html( + "The ||dog dies at the end of Marley and Me||.\n", + "
The dog dies at the end of Marley and Me.
\n", + ); +} + +#[test] +fn spoiler_in_table() { + html( + "Text | Result\n--- | ---\n`||some clever text||` | ||some clever text||\n", + concat!( + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
TextResult
||some clever text||some clever text
\n" + ), + ); +} + +#[test] +fn spoiler_regressions() { + html( + "|should not be spoiler|\n||should be spoiler||\n|||should be spoiler surrounded by pipes|||", + concat!( + "
|should not be spoiler|
\n", + "should be spoiler
\n", + "|should be spoiler surrounded by pipes|
\n" + ), + ); +} + +#[test] +fn mismatched_spoilers() { + html( + "|||this is a spoiler with pipe in front||\n||this is not a spoiler|\n||this is a spoiler with pipe after|||", + concat!( + "
|this is a spoiler with pipe in front
\n", + "||this is not a spoiler|
\n", + "this is a spoiler with pipe after|
\n" + ), + ); +} + +#[test] +fn underline() { + html( + "__underlined__\n", + "
underlined
\n", + ); +} + +#[test] +fn no_setext_headings_in_philomena() { + html( + "text text\n---", + "
text text
\n
\n", + ); +} + +#[test] +fn greentext_preserved() { + html( + ">implying\n>>implying", + "
>implying
\n>>implying
\n", + ); +} + +#[test] +fn separate_quotes_on_line_end() { + html( + "> 1\n>\n> 2", + "
\n
1
\n
\n
>
\n
\n
2
\n
\n" + ); +} + +#[test] +fn unnest_quotes_on_line_end() { + html( + "> 1\n> > 2\n> 1", + "
\n
1
\n
\n
2
\n
\n
1
\n
\n", + ); +} + +#[test] +fn unnest_quotes_on_line_end_commonmark() { + html( + "> 1\n> > 2\n> \n> 1", + "
\n
1
\n
\n
2
\n
\n
1
\n
\n", + ); +} + +#[test] +fn philomena_images() { + html( + "![full](http://example.com/image.png)", + "
\"full\"
\n", + ); +} + +#[test] +fn no_empty_link() { + html_opts_i( + "[](https://example.com/evil.domain.for.seo.spam)", + "
[](https://example.com/evil.domain.for.seo.spam)
\n", + |opts| opts.extension.autolink = false, + ); + + html_opts_i( + "[ ](https://example.com/evil.domain.for.seo.spam)", + "
[ ](https://example.com/evil.domain.for.seo.spam)
\n", + |opts| opts.extension.autolink = false, + ); +} + +#[test] +fn empty_image_allowed() { + html( + "![ ](https://example.com/evil.domain.for.seo.spam)", + "
\"
\n", + ); +} + +#[test] +fn image_inside_link_allowed() { + html( + "[![](https://example.com/image.png)](https://example.com/)", + "
\"\"
\n", + ); +} + +#[test] +fn image_mention() { + html_opts_i( + "hello world >>1234p >>1337", + "
hello world
p
>>1337
\n", + |opts| { + let mut replacements = HashMap::new(); + replacements.insert("1234p".to_string(), "
p
".to_string()); + + opts.extension.replacements = Some(replacements); + }, + ); +} + +#[test] +fn image_mention_line_start() { + html_opts_i( + ">>1234p", + "
p
\n", + |opts| { + let mut replacements = HashMap::new(); + replacements.insert("1234p".to_string(), "
p
".to_string()); + + opts.extension.replacements = Some(replacements); + }, + ); +} + +#[test] +fn auto_relative_links() { + let domains: Vec = vec!["example.com".into()]; + let f = Arc::new(move |url: &str| domains::relativize_careful(&domains, url)); + + html_opts_i( + "[some link text](https://example.com/some/path)", + "
some link text
\n", + |opts| { + opts.extension.link_url_rewriter = Some(f.clone()); + }, + ); + + html_opts_i( + "https://example.com/some/path", + "
https://example.com/some/path
\n", + |opts| { + opts.extension.link_url_rewriter = Some(f.clone()); + }, + ); + + html_opts_i( + "[some link text](https://example.com/some/path?parameter=aaaaaa&other_parameter=bbbbbb#id12345)", + "
some link text
\n", + |opts| { + opts.extension.link_url_rewriter = Some(f.clone()); + }, + ); + + html_opts_i( + "https://example.com/some/path?parameter=aaaaaa&other_parameter=bbbbbb#id12345", + "
https://example.com/some/path?parameter=aaaaaa&other_parameter=bbbbbb#id12345
\n", + |opts| { + opts.extension.link_url_rewriter = Some(f.clone()); + }, + ); +}