From 2bd690dd16b44ece3b2ae6d392e996a0795474cf Mon Sep 17 00:00:00 2001 From: Thomas Montague Date: Mon, 10 Jun 2024 13:11:51 -0500 Subject: [PATCH] initial commit. --- .dockerignore | 2 + .github/workflows/docker-publish.yml | 101 +++++++++++++ .gitignore | 2 + Dockerfile | 7 + LICENSE | 21 +++ README.md | 14 ++ download.py | 164 +++++++++++++++++++++ requirements.txt | 4 + upload.py | 204 +++++++++++++++++++++++++++ 9 files changed, 519 insertions(+) create mode 100644 .dockerignore create mode 100644 .github/workflows/docker-publish.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 README.md create mode 100755 download.py create mode 100644 requirements.txt create mode 100755 upload.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6e19512 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.dockerignore +Dockerfile diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..ce905ad --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,101 @@ +name: Docker + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +on: + schedule: + - cron: '42 17 * * *' + push: + branches: [ "main" ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + pull_request: + branches: [ "main" ] + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + + +jobs: + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0 + with: + cosign-release: 'v2.2.4' + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + # set latest tag for default branch + type=raw,value=latest,enable={{is_default_branch}} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6661d5b --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.vscode +/storage diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7d9793d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3-alpine +RUN apk upgrade --no-cache +WORKDIR /app +COPY requirements.txt /app/ +RUN pip install --no-cache-dir --upgrade -r requirements.txt +COPY . /app/ +VOLUME /app/storage diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c8c74ed --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Ash Wilson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..f2fd2da --- /dev/null +++ b/README.md @@ -0,0 +1,14 @@ +# Slackmojis + +Downloads emojis from [slackmojis.com](https://slackmojis.com) and generates "packs" by +category. Allows uploading of emojis to Slack as a normal Slack user with permissions +to add emojis. + +## Usage + + + +## References + +- https://github.com/lambtron/emojipacks +- https://github.com/smashwilson/slack-emojinator diff --git a/download.py b/download.py new file mode 100755 index 0000000..b291e78 --- /dev/null +++ b/download.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +import functools, io, json, multiprocessing, os, requests, yaml +from PIL import Image + + +SLACKMOJIS_JSON_URL = "https://slackmojis.com/emojis.json" +STORAGE_DIR = "storage" +DOWNLOAD_DIR = "downloaded" +PACK_DIR = "packs" +SLACKMOJIS_JSON_FILE = "slackmojis.json" +PARALLELISM = multiprocessing.cpu_count() - 1 + + +def create_dirs(dir): + """Create directory and all intermediate-level directories""" + if not os.path.isdir(dir): + os.makedirs(dir) + +def download_file(url, output_file): + response = requests.get(url) + with open(output_file, "wb") as f: + f.write(response.content) + return response + +def write_yaml_file(data, output_file): + with open(output_file, "w") as f: + yaml.dump(data, f, default_flow_style=False) + +def get_slackmojis(url, output_file): + if os.path.isfile(output_file): + with open(output_file, "r") as f: + return json.load(f) + + print("fetching slackmojis ...") + page=0 + slackmojis = [] + while True: + print(f"... page {page}") + response = requests.get(url + f"?page={page}") + response.raise_for_status() + emojis = response.json() + if len(emojis) == 0: + break + slackmojis.extend(emojis) + page = page + 1 + with open(output_file, "w") as f: + json.dump(slackmojis, f) + print("done.") + return slackmojis + +def get_categories(slackmojis): + categories = set() + categories.add("uncategorized") + for slackmoji in slackmojis: + if "category" in slackmoji: + category = slackmoji["category"]["name"].lower().replace(" ", "-") + categories.add(category) + return categories + +def valid_image(name, src): + try: + ext = os.path.splitext(src)[1] + # the downloaded filename is different from if you download it manually + # because of the possible duplicates + dl_file = os.path.join(STORAGE_DIR, DOWNLOAD_DIR, f"{name}{ext}") + if os.path.isfile(dl_file): + with open(dl_file, "rb") as f: + body = f.read() + else: + response = download_file(src, dl_file) + body = response.content + + with io.BytesIO(body) as f: + # Is it an image? + im = Image.open(f) + if im.width > 256 or im.height > 256: + print(f":{name}: ({dl_file}) is {im.size}\t{src}") + return False, None + + return True, dl_file + except Exception as e: + print(f":{name}: ({dl_file}) - unknown exception - {e}") + return False, None + + +def process_slackmoji(slackmoji, name_count, packs): + name = slackmoji["name"] + print(f"... {name}") + + category = "uncategorized" + if "category" in slackmoji: + category = slackmoji["category"]["name"].lower().replace(" ", "-") + + # Special cases - a.k.a stupid cases + if name == "yes2": + # there are two "yes" and one "yes2" emojis already + name = "yes2-1" + if name == "no2": + # there are two "no" and one "no2" emojis already + name = "no2-1" + sports = ["mlb", "nba", "nfl", "nhl"] + if category in sports: + # The NFL logo should not be :nfl-nfl: + if name == "nfl": + pass + else: + name = f"{category}-{name}" + if "facebook" in category: + name = f"fb-{name}" + if "scrabble" in category: + name = f"scrabble-{name}" + + name_count[name] = name_count[name] + 1 if name in name_count else 1 + if name_count[name] > 1: + name = f"{name}{name_count[name]}" + src = slackmoji["image_url"].split("?")[0] + + success, dl_file = valid_image(name, src) + if not success: + return + + packs[category]["emojis"].append({ + "name": name, + "file": dl_file, + "src": src, + }) + + +def main(): + slackmoji_pack_dir = os.path.join(STORAGE_DIR, PACK_DIR) + create_dirs(slackmoji_pack_dir) + create_dirs(os.path.join(STORAGE_DIR, DOWNLOAD_DIR)) + + slackmojis = get_slackmojis(SLACKMOJIS_JSON_URL, os.path.join(STORAGE_DIR, SLACKMOJIS_JSON_FILE)) + categories = get_categories(slackmojis) + + with multiprocessing.Manager() as manager: + # Initialize dicts + name_count = manager.dict() + packs = manager.dict() + for category in categories: + packs[category] = { + "title": f"slackmoji-{category}", + "emojis": manager.list(), + } + + # Process slackmojis + with multiprocessing.Pool(processes=PARALLELISM) as pool: + print("processing slackmojis ...") + _process_slackmoji = functools.partial(process_slackmoji, name_count=name_count, packs=packs) + pool.map(_process_slackmoji, slackmojis) + print("done") + + print("writing category files ...") + for category in categories: + print(f"... {category}") + data = packs[category] + data["emojis"] = list(data["emojis"]) + write_yaml_file(data, os.path.join(slackmoji_pack_dir, f"slackmojis-{category}.yaml")) + print("done") + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b7ed864 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +beautifulsoup4 +pillow +pyyaml +requests diff --git a/upload.py b/upload.py new file mode 100755 index 0000000..3256044 --- /dev/null +++ b/upload.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 + +# Upload files named on ARGV as Slack emoji. +# https://github.com/smashwilson/slack-emojinator + +import argparse +import os, re, requests, sys +from bs4 import BeautifulSoup +from time import sleep + + +try: + raw_input +except NameError: + raw_input = input + +URL_CUSTOMIZE = "https://{team_name}.slack.com/customize/emoji" +URL_ADD = "https://{team_name}.slack.com/api/emoji.add" +URL_LIST = "https://{team_name}.slack.com/api/emoji.adminList" + +API_TOKEN_REGEX = r'.*(?:\"?api_token\"?):\s*\"([^"]+)\".*' +API_TOKEN_PATTERN = re.compile(API_TOKEN_REGEX) + +JAVASCRIPT_MATCH_TOKEN = ''' +javascript:prompt("Your emoji API Token is:", document.documentElement.innerHTML.match('.*(?:\"?api_token\"?):\s*\"([^"]+)\".*')[1]) +''' + + +class ParseError(Exception): + pass + + +def _session(args): + assert args.cookie, "Cookie required" + assert args.team_name, "Team name required" + session = requests.session() + session.headers = { + "Cookie": f"d={args.cookie}", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36", + } + session.url_customize = URL_CUSTOMIZE.format(team_name=args.team_name) + session.url_add = URL_ADD.format(team_name=args.team_name) + session.url_list = URL_LIST.format(team_name=args.team_name) + session.api_token = _fetch_api_token(session) + return session + + +def _argparse(): + parser = argparse.ArgumentParser( + description="Bulk upload emoji to slack" + ) + parser.add_argument( + "--team-name", "-t", + default=os.getenv("SLACK_TEAM"), + help="Defaults to the $SLACK_TEAM environment variable." + ) + parser.add_argument( + "--cookie", "-c", + default=os.getenv("SLACK_COOKIE"), + help="Defaults to the $SLACK_COOKIE environment variable." + ) + parser.add_argument( + "--prefix", "-p", + default=os.getenv("EMOJI_NAME_PREFIX", ""), + help="Prefix to add to genereted emoji name. " + "Defaults to the $EMOJI_NAME_PREFIX environment variable." + ) + parser.add_argument( + "--suffix", "-s", + default=os.getenv("EMOJI_NAME_SUFFIX", ""), + help="Suffix to add to generated emoji name. " + "Defaults to the $EMOJI_NAME_SUFFIX environment variable." + ) + parser.add_argument( + "slackmoji_files", + nargs="+", + help=("Paths to slackmoji, e.g. if you " + "unzipped http://cultofthepartyparrot.com/parrots.zip " + "in your home dir, then use ~/parrots/*"), + ) + args = parser.parse_args() + if not args.team_name: + args.team_name = raw_input("Please enter the team name: ").strip() + if not args.cookie: + args.cookie = raw_input(f"Please enter the \"d\" cookie value from slack (https://{args.team_name}.slack.com/customize/emoji): ").strip() + return args + + +def _fetch_api_token(session): + # Fetch the form first, to get an api_token. + r = session.get(session.url_customize) + r.raise_for_status() + soup = BeautifulSoup(r.text, "html.parser") + + all_script = soup.findAll("script") + for script in all_script: + for line in script.text.splitlines(): + if '"api_token"' in line: + # api_token: "xoxs-12345-abcdefg....", + # "api_token":"xoxs-12345-abcdefg....", + match_group = API_TOKEN_PATTERN.match(line.strip()) + if not match_group: + print(line) + raise ParseError( + "Could not parse API token from remote data! " + "Regex requires updating." + ) + + return match_group.group(1) + + print(f"No api_token found in page. Search your {session.url_customize} " + "page source for \"api_token\" and enter its value manually.") + return raw_input("Please enter the api_token (\"xoxs-12345-abcdefg....\") from the page: ").strip() + + +def main(): + args = _argparse() + session = _session(args) + existing_emojis = get_current_emoji_list(session) + uploaded = 0 + skipped = 0 + + def process_file(filename): + nonlocal skipped + nonlocal uploaded + print("Processing {}.".format(filename)) + emoji_name = "{}{}{}".format( + args.prefix.strip(), + os.path.splitext(os.path.basename(filename))[0], + args.suffix.strip() + ) + if emoji_name in existing_emojis: + print("Skipping {}. Emoji already exists".format(emoji_name)) + skipped += 1 + else: + upload_emoji(session, emoji_name, filename) + print("{} upload complete.".format(filename)) + uploaded += 1 + + for slackmoji_file in args.slackmoji_files: + if os.path.isdir(slackmoji_file): + for file in os.listdir(slackmoji_file): + filename = os.path.join(slackmoji_file, file) + process_file(filename) + else: + process_file(slackmoji_file) + print("\nUploaded {} emojis. ({} already existed)".format(uploaded, skipped)) + + +def get_current_emoji_list(session): + page = 1 + result = [] + while True: + data = { + "query": "", + "page": page, + "count": 1000, + "token": session.api_token + } + resp = session.post(session.url_list, data=data) + resp.raise_for_status() + response_json = resp.json() + if not response_json["ok"]: + print("Error with fetching emoji list: %s" % (response_json)) + sys.exit(1) + + result.extend(map(lambda e: e["name"], response_json["emoji"])) + if page >= response_json["paging"]["pages"]: + break + + page = page + 1 + return result + + +def upload_emoji(session, emoji_name, filename): + data = { + "mode": "data", + "name": emoji_name, + "token": session.api_token + } + + while True: + with open(filename, "rb") as f: + files = {"image": f} + resp = session.post(session.url_add, data=data, files=files, allow_redirects=False) + + if resp.status_code == 429: + wait = int(resp.headers.get("retry-after", 1)) + print("429 Too Many Requests!, sleeping for %d seconds" % wait) + sleep(wait) + continue + + resp.raise_for_status() + + # Slack returns 200 OK even if upload fails, so check for status. + response_json = resp.json() + if not response_json["ok"]: + print("Error with uploading %s: %s" % (emoji_name, response_json)) + + break + + +if __name__ == "__main__": + main()