Skip to content

Commit

Permalink
Publish libraries on smithed (#889)
Browse files Browse the repository at this point in the history 
* Publish libraries to a folder in release

Standalone library build, with needed lantern load files, saved to the release directory to be uploaded to smithed

* Version() support for rich comparison operators

* Modify release script for libraries

* Auto patch tracking for libraries

* Update manifest.py

* Release libraries with module release plugin

* Start to version permalinking

* Complete past-version permalinking

* Fix lantern-load inclusion

* beet-release.yaml plugins for LL

* Library Licenses and Readmes

* Version number appears in lib.zip pack.mcmeta

* Rewrite lib_brewing README

- actually offers explanations for things
- restructures "Additional Uses" section

* Simple typos

* potion tracking clarification

- adds example functions to show how to tag potions and how to execute from landed potions

* custom crafters example

- add loot table for example recipe
- switch example to 1 wool -> 3 string to explain that rounding down is necessary

* Port over example-use packs

* Add smithed pack ids

* Simple library pack icons

Simple inkscape-bitmap-traced icons. Do not conform with the GM4 color map and should be adjusted in the future. I blindly used the colorize filter to achieve this effect

* Copy READMEs to the generated directory

---------

Co-authored-by: BluePsychoRanger <[email protected]>
  • Loading branch information
@SpecialBuilder32 @BPR02
SpecialBuilder32 and BPR02 authored Oct 4, 2023
1 parent b1b6eec commit f6e3aff
Show file tree
Hide file tree
Showing 123 changed files with 3,033 additions and 274 deletions.
24 changes: 23 additions & 1 deletion beet-release.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,34 @@ pipeline:
meta: {copy_files: {data_pack: {LICENSE.md: "../LICENSE.md"}}}
- gm4.plugins.readme_generator
- gm4.plugins.write_mcmeta
- gm4.plugins.output.release
meta:
mecha:
formatting:
layout: preserve
nbt_compact: True
- broadcast: 'lib_*'
extend: 'beet.yaml'
pipeline:
- beet.contrib.lantern_load.base_data_pack
- gm4.plugins.versioning.isolated_library
- gm4.plugins.manifest.write_credits
- require: [beet.contrib.copy_files]
meta:
copy_files:
data_pack:
LICENSE.md: "LICENSE.md"
README.md: "README.md"
pack.png: "pack.png"
- gm4.plugins.module.default_pack_icon
- gm4.plugins.readme_generator.libraries
- gm4.plugins.write_mcmeta
- gm4.plugins.output.release
meta:
mecha:
formatting:
layout: preserve
nbt_compact: True

meta:
autosave:
link: false
88 changes: 46 additions & 42 deletions gm4/plugins/manifest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,8 @@ class ManifestCacheModel(BaseModel):
class ManifestFileModel(BaseModel):
"""describes the structure of the meta.json saved to disk"""
last_commit: str
modules: list[ManifestModuleModel]
modules: list[ManifestModuleModel] # straight list for website backward compat
libraries: dict[str, ManifestModuleModel]
contributors: Any


Expand Down Expand Up @@ -136,59 +137,63 @@ def update_patch(ctx: Context):
logger = parent_logger.getChild("update_patch")
skin_cache = JsonFile(source_path="gm4/skin_cache.json").data

modules = ManifestCacheModel.parse_obj(ctx.cache["gm4_manifest"].json).modules
manifest_cache = ManifestCacheModel.parse_obj(ctx.cache["gm4_manifest"].json)

if manifest_file.exists():
manifest = ManifestFileModel.parse_obj(json.loads(manifest_file.read_text()))
last_commit = manifest.last_commit
released_modules: dict[str, ManifestModuleModel] = {m.id:m for m in manifest.modules if m.version}
released_modules |= manifest.libraries
else:
logger.debug("No existing meta.json manifest file was located")
last_commit = None
released_modules = {}

for id in modules:
module = modules[id]
released = released_modules.get(id, None)

publish_date = released.publish_date if released else None
module.publish_date = publish_date or datetime.datetime.now().date().isoformat()

deps = _traverse_includes(id) | {"base"}
deps_dirs = [element for sublist in [[f"{d}/data", f"{d}/*py"] for d in deps] for element in sublist]

# add watches to skins this module uses from other modules. NOTE this could be done in a more extendable way in the future, rather than "hardcoded"
skin_dep_dirs: list[str] = []
for skin_ref in skin_cache["nonnative_references"].get(id, []):
d = skin_cache["skins"][skin_ref]["parent_module"]
ns, path = skin_ref.split(":")
skin_dep_dirs.append(f"{d}/data/{ns}/skins/{path}.png")

watch_dirs = deps_dirs+skin_dep_dirs

diff = run(["git", "diff", last_commit, "--shortstat", "--", f"{id}/data", f"{id}/*.py"] + watch_dirs) if last_commit else True

if not diff and released:
# No changes were made, keep the same patch version
module.version = released.version
elif not released:
# First release
module.version = module.version.replace("X", "0")
logger.debug(f"First release of {id}")
else:
# Changes were made, bump the patch
version = Version(module.version)
last_ver = Version(released.version)
for packs in (manifest_cache.modules, manifest_cache.libraries):
for id in packs:
pack = packs[id]
released = released_modules.get(id, None)

publish_date = released.publish_date if released else None
pack.publish_date = publish_date or datetime.datetime.now().date().isoformat()

deps = _traverse_includes(id)
if packs is manifest_cache.modules:
deps |= {"base"} # scan the base directory if this is a module
deps_dirs = [element for sublist in [[f"{d}/data", f"{d}/*py"] for d in deps] for element in sublist]

# add watches to skins this module uses from other modules. NOTE this could be done in a more extendable way in the future, rather than "hardcoded"
skin_dep_dirs: list[str] = []
for skin_ref in skin_cache["nonnative_references"].get(id, []):
d = skin_cache["skins"][skin_ref]["parent_module"]
ns, path = skin_ref.split(":")
skin_dep_dirs.append(f"{d}/data/{ns}/skins/{path}.png")

if version.minor > last_ver.minor or version.major > last_ver.major: # type: ignore
version.patch = 0
watch_dirs = deps_dirs+skin_dep_dirs

diff = run(["git", "diff", last_commit, "--shortstat", "--", f"{id}/data", f"{id}/*.py"] + watch_dirs) if last_commit else True

if not diff and released:
# No changes were made, keep the same patch version
pack.version = released.version
elif not released:
# First release
pack.version = pack.version.replace("X", "0")
logger.debug(f"First release of {id}")
else:
version.patch = last_ver.patch + 1 # type: ignore
logger.info(f"Updating {id} patch to {version.patch}")
# Changes were made, bump the patch
version = Version(pack.version)
last_ver = Version(released.version)

if version.minor > last_ver.minor or version.major > last_ver.major: # type: ignore
version.patch = 0
else:
version.patch = last_ver.patch + 1 # type: ignore
logger.info(f"Updating {id} patch to {version.patch}")

module.version = str(version)
pack.version = str(version)

ctx.cache["gm4_manifest"].json["modules"] = {id:m.dict() for id, m in modules.items()}
ctx.cache["gm4_manifest"].json = manifest_cache.dict()

@cache
def _traverse_includes(project_id: str) -> set[str]:
Expand All @@ -213,7 +218,6 @@ def write_meta(ctx: Context):
manifest_file = release_dir / "meta.json"
manifest = ctx.cache["gm4_manifest"].json.copy()
manifest["modules"] = list(manifest["modules"].values()) # convert modules dict down to list for backwards compatability
manifest.pop("libraries")
manifest.pop("base")
manifest_file.write_text(json.dumps(manifest, indent=2))

Expand Down Expand Up @@ -260,7 +264,7 @@ def write_updates(ctx: Context):
if init is None:
return

manifest =ManifestCacheModel.parse_obj(ctx.cache["gm4_manifest"].json)
manifest = ManifestCacheModel.parse_obj(ctx.cache["gm4_manifest"].json)
modules = manifest.modules

score = f"{ctx.project_id.removeprefix('gm4_')} gm4_modules"
Expand Down
119 changes: 69 additions & 50 deletions gm4/plugins/output.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
import requests
import shutil
import logging
from gm4.utils import run
from gm4.utils import run, Version, NoneAttribute
import gm4.plugins.manifest # for ManifestCacheModel; a runtime circular dependency

parent_logger = logging.getLogger("gm4.output")

Expand All @@ -19,9 +20,11 @@

class ModrinthConfig(PluginOptions):
project_id: Optional[str]
minecraft: list[str] = SUPPORTED_GAME_VERSIONS

class SmithedConfig(PluginOptions):
pack_id: Optional[str]
minecraft: list[str] = SUPPORTED_GAME_VERSIONS

class PMCConfig(PluginOptions):
uid: Optional[int]
Expand Down Expand Up @@ -55,7 +58,10 @@ def release(ctx: Context):
"""
version_dir = os.getenv("VERSION", "1.20")
release_dir = Path("release") / version_dir
file_name = f"{ctx.project_id}_{version_dir.replace('.', '_')}.zip"

corrected_project_id = stem if (stem:=ctx.directory.stem).startswith("lib") else ctx.project_id

file_name = f"{corrected_project_id}_{version_dir.replace('.', '_')}.zip"

yield # wait for exit phase, after other plugins cleanup

Expand All @@ -70,37 +76,36 @@ def release(ctx: Context):
pack_icon_dir = generated_dir / "pack_icons"
os.makedirs(pack_icon_dir, exist_ok=True)
if "pack.png" in ctx.data.extra:
ctx.data.extra["pack.png"].dump(pack_icon_dir, f"{ctx.project_id}.png")
ctx.data.extra["pack.png"].dump(pack_icon_dir, f"{corrected_project_id}.png")

smithed_readme_dir = generated_dir / "smithed_readmes"
os.makedirs(smithed_readme_dir, exist_ok=True)
if "smithed_readme" in ctx.meta:
ctx.meta['smithed_readme'].dump(smithed_readme_dir, f"{ctx.project_id}.md")
ctx.meta['smithed_readme'].dump(smithed_readme_dir, f"{corrected_project_id}.md")

# publish to download platforms
publish_modrinth(ctx, release_dir, file_name)
publish_smithed(ctx, release_dir, file_name)
publish_smithed(ctx, file_name)


def publish_modrinth(ctx: Context, release_dir: Path, file_name: str):
'''Attempts to publish pack to modrinth'''
modrinth = ctx.meta.get("modrinth", None)
opts = ctx.validate("modrinth", ModrinthConfig)
auth_token = os.getenv(MODRINTH_AUTH_KEY, None)
logger = parent_logger.getChild(f"modrinth.{ctx.project_id}")
if modrinth and auth_token:
modrinth_id = modrinth["project_id"]

if opts.project_id and auth_token:
# update page description
res = requests.get(f"{MODRINTH_API}/project/{modrinth_id}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT})
res = requests.get(f"{MODRINTH_API}/project/{opts.project_id}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT})
if not (200 <= res.status_code < 300):
if res.status_code == 404:
logger.warning(f"Cannot edit description of modrinth project {modrinth_id} as it doesn't exist.")
logger.warning(f"Cannot edit description of modrinth project {opts.project_id} as it doesn't exist.")
else:
logger.warning(f"Failed to get project: {res.status_code} {res.text}")
return
existing_readme = res.json()["body"]
if existing_readme != (d:=ctx.meta['modrinth_readme'].text):
logger.debug("Readme and modrinth-page content differ. Updating webpage body")
res = requests.patch(f"{MODRINTH_API}/project/{modrinth_id}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}, json={"body": d})
res = requests.patch(f"{MODRINTH_API}/project/{opts.project_id}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}, json={"body": d})
if not (200 <= res.status_code < 300):
logger.warning(f"Failed to update description: {res.status_code} {res.text}")
logger.info(f"Successfully updated description of {ctx.project_name}")
Expand All @@ -112,16 +117,16 @@ def publish_modrinth(ctx: Context, release_dir: Path, file_name: str):
logger.warning("Full version number not available in ctx.meta. Skipping publishing")
return

res = requests.get(f"{MODRINTH_API}/project/{modrinth_id}/version", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT})
res = requests.get(f"{MODRINTH_API}/project/{opts.project_id}/version", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT})
if not (200 <= res.status_code < 300):
if res.status_code == 404:
logger.warning(f"Cannot publish to modrinth project {modrinth_id} as it doesn't exist.")
logger.warning(f"Cannot publish to modrinth project {opts.project_id} as it doesn't exist.")
else:
logger.warning(f"Failed to get project versions: {res.status_code} {res.text}")
return
project_data = res.json()
matching_version = next((v for v in project_data if v["version_number"] == str(version)), None)
game_versions = modrinth.get("minecraft", SUPPORTED_GAME_VERSIONS)
game_versions = opts.minecraft
if matching_version is not None: # patch version already exists
# update mc versions if necessary
if not set(matching_version["game_versions"]) == set(game_versions):
Expand All @@ -148,7 +153,7 @@ def publish_modrinth(ctx: Context, release_dir: Path, file_name: str):
"version_type": "release",
"loaders": ["datapack"],
"featured": False,
"project_id": modrinth_id,
"project_id": opts.project_id,
"file_parts": [file_name],
}),
file_name: file_bytes,
Expand All @@ -158,54 +163,58 @@ def publish_modrinth(ctx: Context, release_dir: Path, file_name: str):
return
logger.info(f"Successfully published {res.json()['name']}")

def publish_smithed(ctx: Context, release_dir: Path, file_name: str):
def publish_smithed(ctx: Context, file_name: str):
"""Attempts to publish pack to smithed"""
smithed = ctx.meta.get("smithed", None)
opts = ctx.validate("smithed", SmithedConfig)
auth_token = os.getenv(SMITHED_AUTH_KEY, None)
logger = parent_logger.getChild(f"smithed.{ctx.project_id}")
mc_version_dir = os.getenv("VERSION", "1.20")
if smithed and auth_token and ctx.project_version:
version = ctx.cache["gm4_manifest"].json["modules"].get(ctx.project_id, {}).get("version", None)
smithed_id = smithed["pack_id"]
manifest = gm4.plugins.manifest.ManifestCacheModel.parse_obj(ctx.cache["gm4_manifest"].json)
project_id = stem if (stem:=ctx.directory.stem).startswith("lib") else ctx.project_id

if opts.pack_id and auth_token:
version = (manifest.modules|manifest.libraries).get(project_id, NoneAttribute()).version or ""

# get project data and existing versions
res = requests.get(f"{SMITHED_API}/packs/{smithed_id}")
res = requests.get(f"{SMITHED_API}/packs/{opts.pack_id}")
if not (200 <= res.status_code < 300):
if res.status_code == 404:
logger.warning(f"Cannot publish to smithed project {smithed_id} as it doesn't exist.")
logger.warning(f"Cannot publish to smithed project {opts.pack_id} as it doesn't exist.")
else:
logger.warning(f"Failed to get project: {res.status_code} {res.text}")
return

project_data = res.json()

# update description and pack image
# ensures they point to the most up-to-date mc version branch
project_display = project_data["display"]
current_icon = f"https://raw.githubusercontent.com/Gamemode4Dev/GM4_Datapacks/release/{mc_version_dir}/generated/pack_icons/{ctx.project_id}.png"
current_readme = f"https://raw.githubusercontent.com/Gamemode4Dev/GM4_Datapacks/release/{mc_version_dir}/generated/smithed_readmes/{ctx.project_id}.md"

if project_display["icon"] != current_icon or project_display["webPage"] != current_readme:
logger.debug("Pack Icon or Readme hyperlink is incorrect. Updating project")
res = requests.patch(f"{SMITHED_API}/packs/{smithed_id}", params={'token': auth_token},
json={"data": {
"display": {
"icon": current_icon,
"webPage": current_readme,
},
}})
if not (200 <= res.status_code < 300):
logger.warning(f"Failed to update descripion: {res.status_code} {res.text}")
logger.info(f"{ctx.project_name} {res.text}")

project_versions = project_data["versions"]
newest_version = sorted([Version(v["name"]) for v in project_versions])[-1]
if Version(version) > newest_version: # only update the description if we're not patching an old version
project_display = project_data["display"]
current_icon = f"https://raw.githubusercontent.com/Gamemode4Dev/GM4_Datapacks/release/{mc_version_dir}/generated/pack_icons/{project_id}.png"
current_readme = f"https://raw.githubusercontent.com/Gamemode4Dev/GM4_Datapacks/release/{mc_version_dir}/generated/smithed_readmes/{project_id}.md"

if project_display["icon"] != current_icon or project_display["webPage"] != current_readme:
logger.debug("Pack Icon or Readme hyperlink is incorrect. Updating project")
res = requests.patch(f"{SMITHED_API}/packs/{opts.pack_id}", params={'token': auth_token},
json={"data": {
"display": {
"icon": current_icon,
"webPage": current_readme,
},
}})
if not (200 <= res.status_code < 300):
logger.warning(f"Failed to update descripion: {res.status_code} {res.text}")
logger.info(f"{ctx.project_name} {res.text}")

matching_version = next((v for v in project_versions if v["name"] == str(version)), None)
game_versions = smithed.get("minecraft", SUPPORTED_GAME_VERSIONS)
game_versions = opts.minecraft
if matching_version is not None: # patch version already exists
# update MC version if necessary
if not set(matching_version["supports"]) == set(game_versions):
logger.debug("Additional MC version support has been added to an existing patch version. Updating existing smithed version data")
res = requests.patch(f"{SMITHED_API}/packs/{smithed_id}/versions/{matching_version['name']}", params={'token': auth_token}, json={
res = requests.patch(f"{SMITHED_API}/packs/{opts.pack_id}/versions/{matching_version['name']}", params={'token': auth_token}, json={
"data": {
"supports": game_versions
}
Expand All @@ -214,17 +223,27 @@ def publish_smithed(ctx: Context, release_dir: Path, file_name: str):
logger.warning(f"Failed to patch project versions: {res.status_code} {res.text}")
return

# remove other existing versions for that mc version
mc_version_matching_version = (v["name"] for v in project_versions if set(v['supports']) & set(game_versions))
for v in mc_version_matching_version:
res = requests.delete(f"{SMITHED_API}/packs/{smithed_id}/versions/{v}", params={'token': auth_token})
# permalink previous version (in that MC version) to the git history
commit_hash = run("cd release && git log -1 --format=%H")
matching_mc_versions = sorted((Version(v["name"]) for v in project_versions if set(v['supports']) & set(game_versions)))
prior_version_in_mc_version = matching_mc_versions[-1] if len(matching_mc_versions) > 0 else None # newest version number, with any MC overlap
prior_url: str = next((v["downloads"]["datapack"] for v in project_versions if Version(v["name"]) == prior_version_in_mc_version), "")
if "https://github.com/Gamemode4Dev/GM4_Datapacks/blob/" not in prior_url and prior_version_in_mc_version:
res = requests.patch(f"{SMITHED_API}/packs/{opts.pack_id}/versions/{prior_version_in_mc_version}", params={'token': auth_token}, json={
"data":{
"downloads": {
"datapack": f"https://github.com/Gamemode4Dev/GM4_Datapacks/blob/{commit_hash}/{mc_version_dir}/{file_name}?raw=true",
"resourcepack": ""
}
}
})
if not (200 <= res.status_code < 300):
logger.warning(f"Failed to delete {ctx.project_name} version {v}: {res.status_code} {res.text}")
logger.warning(f"Failed to permalink {project_id} version {prior_version_in_mc_version}: {res.status_code} {res.text}")
else:
logger.info(f"{ctx.project_name} {res.text}")
logger.info(f"Permalinked {project_id} {prior_version_in_mc_version} to git history: {res.text}")

# post new version
res = requests.post(f"{SMITHED_API}/packs/{smithed_id}/versions",
res = requests.post(f"{SMITHED_API}/packs/{opts.pack_id}/versions",
params={'token': auth_token, 'version': version},
json={"data":{
"downloads":{
Expand Down
Loading

0 comments on commit f6e3aff

Please sign in to comment.