From 39460b49be4311b9c0f663c899d6fa0609debeb3 Mon Sep 17 00:00:00 2001 From: Marin Minnerly Date: Wed, 20 Nov 2024 15:15:19 -0800 Subject: [PATCH 1/6] Publishing works! --- .gitignore | 3 + .lune/lib/run.luau | 13 +- .lune/open-cloud/luau_execution_task.py | 272 ++++++++++++++++++++++++ .lune/open-cloud/upload_and_run_task.py | 69 ++++++ .lune/publish-plugin.luau | 51 +++++ .lune/tasks/publish-plugin-asset.luau | 40 ++++ project.luau | 8 + 7 files changed, 450 insertions(+), 6 deletions(-) create mode 100644 .lune/open-cloud/luau_execution_task.py create mode 100644 .lune/open-cloud/upload_and_run_task.py create mode 100644 .lune/publish-plugin.luau create mode 100644 .lune/tasks/publish-plugin-asset.luau create mode 100644 project.luau diff --git a/.gitignore b/.gitignore index e714241d..e6d8884c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,8 @@ DevPackages Packages wally.lock +# Python +__pycache__ + # Other /*.log diff --git a/.lune/lib/run.luau b/.lune/lib/run.luau index 357c3e79..6d2dc3b6 100644 --- a/.lune/lib/run.luau +++ b/.lune/lib/run.luau @@ -18,15 +18,16 @@ local function run(program: string, params: { string }, options: Options?) env = if options then options.env else nil, }) - if result.code > 0 then - process.exit(result.code) + local out + if result.ok then + out = result.stdout + else + out = result.stderr end - local output = if result.ok then result.stdout else result.stderr + out = out:gsub("\n$", "") - output = output:gsub("\n$", "") - - return output + return out, result.ok end return run diff --git a/.lune/open-cloud/luau_execution_task.py b/.lune/open-cloud/luau_execution_task.py new file mode 100644 index 00000000..31acc64f --- /dev/null +++ b/.lune/open-cloud/luau_execution_task.py @@ -0,0 +1,272 @@ +# upstream: https://raw.githubusercontent.com/Roblox/place-ci-cd-demo/refs/heads/production/scripts/python/luau_execution_task.py + +import argparse +import logging +import urllib.request +import urllib.error +import base64 +import sys +import json +import time +import hashlib +import os + + +def parseArgs(): + parser = argparse.ArgumentParser() + + parser.add_argument( + "-k", + "--api-key", + help="Path to a file containing your OpenCloud API key. You can also use the environment variable RBLX_OC_API_KEY to specify the API key. This option takes precedence over the environment variable.", + metavar="", + ) + parser.add_argument( + "-u", + "--universe", + "-e", + "--experience", + required=True, + help="ID of the experience (a.k.a. universe) containing the place you want to execute the task against.", + metavar="", + type=int, + ) + parser.add_argument( + "-p", + "--place", + required=True, + help="ID of the place you want to execute the task against.", + metavar="", + type=int, + ) + parser.add_argument( + "-v", + "--place-version", + help="Version of the place you want to execute the task against. If not given, the latest version of the place will be used.", + metavar="", + type=int, + ) + parser.add_argument( + "-f", + "--script-file", + required=True, + help="Path to a file containing your Luau script.", + metavar="", + ) + parser.add_argument( + "-c", + "--continuous", + help="If specified, this script will run in a loop and automatically create a new task after the previous task has finished, but only if the script file is updated. If the script file has not been updated, this script will wait for it to be updated before submitting a new task.", + action="store_true", + ) + parser.add_argument( + "-o", + "--output", + help="Path to a file to write the task's output to. If not given, output is written to stdout.", + metavar="", + ) + parser.add_argument( + "-l", + "--log-output", + help="Path to a file to write the task's logs to. If not given, logs are written to stderr.", + metavar="", + ) + + return parser.parse_args() + + +def makeRequest(url, headers, body=None): + data = None + if body is not None: + data = body.encode("utf8") + request = urllib.request.Request( + url, data=data, headers=headers, method="GET" if body is None else "POST" + ) + max_attempts = 3 + for i in range(max_attempts): + try: + return urllib.request.urlopen(request) + except Exception as e: + if "certificate verify failed" in str(e): + logging.error( + f"{str(e)} - you may need to install python certificates, see https://stackoverflow.com/questions/27835619/urllib-and-ssl-certificate-verify-failed-error" + ) + sys.exit(1) + if i == max_attempts - 1: + raise e + else: + logging.info(f"Retrying error: {str(e)}") + time.sleep(1) + + +def readFileExitOnFailure(path, file_description): + try: + with open(path, "r") as f: + return f.read() + except FileNotFoundError: + logging.error(f"{file_description.capitalize()} file not found: {path}") + except IsADirectoryError: + logging.error(f"Invalid {file_description} file: {path} is a directory") + except PermissionError: + logging.error(f"Permission denied to read {file_description} file: {path}") + sys.exit(1) + + +def loadAPIKey(api_key_arg): + source = "" + if api_key_arg: + api_key_arg = api_key_arg.strip() + source = f"file {api_key_arg}" + key = readFileExitOnFailure(api_key_arg, "API key").strip() + else: + if "RBLX_OC_API_KEY" not in os.environ: + logging.error( + "API key needed. Either provide the --api-key option or set the RBLX_OC_API_KEY environment variable." + ) + sys.exit(1) + source = "environment variable RBLX_OC_API_KEY" + key = os.environ["RBLX_OC_API_KEY"].strip() + + try: + base64.b64decode(key, validate=True) + return key + except Exception as e: + logging.error( + f"API key appears invalid (not valid base64, loaded from {source}): {str(e)}" + ) + sys.exit(1) + + +def createTask(api_key, script, universe_id, place_id, place_version): + headers = {"Content-Type": "application/json", "x-api-key": api_key} + data = {"script": script} + url = f"https://apis.roblox.com/cloud/v2/universes/{universe_id}/places/{place_id}/" + if place_version: + url += f"versions/{place_version}/" + url += "luau-execution-session-tasks" + + try: + response = makeRequest(url, headers=headers, body=json.dumps(data)) + except urllib.error.HTTPError as e: + logging.error(f"Create task request failed, response body:\n{e.fp.read()}") + sys.exit(1) + + task = json.loads(response.read()) + return task + + +def pollForTaskCompletion(api_key, path): + headers = {"x-api-key": api_key} + url = f"https://apis.roblox.com/cloud/v2/{path}" + + logging.info("Waiting for task to finish...") + + while True: + try: + response = makeRequest(url, headers=headers) + except urllib.error.HTTPError as e: + logging.error(f"Get task request failed, response body:\n{e.fp.read()}") + sys.exit(1) + + task = json.loads(response.read()) + if task["state"] != "PROCESSING": + sys.stderr.write("\n") + sys.stderr.flush() + return task + else: + sys.stderr.write(".") + sys.stderr.flush() + time.sleep(3) + + +def getTaskLogs(api_key, task_path): + headers = {"x-api-key": api_key} + url = f"https://apis.roblox.com/cloud/v2/{task_path}/logs" + + try: + response = makeRequest(url, headers=headers) + except urllib.error.HTTPError as e: + logging.error(f"Get task logs request failed, response body:\n{e.fp.read()}") + sys.exit(1) + + logs = json.loads(response.read()) + messages = logs["luauExecutionSessionTaskLogs"][0]["messages"] + return "".join([m + "\n" for m in messages]) + + +def handleLogs(task, log_output_file_path, api_key): + logs = getTaskLogs(api_key, task["path"]) + if logs: + if log_output_file_path: + with open(log_output_file_path, "w") as f: + f.write(logs) + logging.info(f"Task logs written to {log_output_file_path}") + else: + logging.info(f"Task logs:\n{logs.strip()}") + else: + logging.info("The task did not produce any logs") + + +def handleSuccess(task, output_path): + output = task["output"] + if output["results"]: + if output_path: + with open(output_path, "w") as f: + f.write(json.dumps(output["results"])) + logging.info(f"Task results written to {output_path}") + else: + logging.info("Task output:") + print(json.dumps(output["results"])) + else: + logging.info("The task did not return any results") + + +def handleFailure(task): + logging.error(f'Task failed, error:\n{json.dumps(task["error"])}') + + +if __name__ == "__main__": + logging.basicConfig( + format="[%(asctime)s] [%(name)s] [%(levelname)s]: %(message)s", + level=logging.INFO, + ) + + args = parseArgs() + + api_key = loadAPIKey(args.api_key) + + waiting_msg_printed = False + prev_script_hash = None + while True: + script = readFileExitOnFailure(args.script_file, "script") + script_hash = hashlib.sha256(script.encode("utf8")).hexdigest() + + if prev_script_hash is not None and script_hash == prev_script_hash: + if not waiting_msg_printed: + logging.info("Waiting for changes to script file...") + waiting_msg_printed = True + time.sleep(1) + continue + + if prev_script_hash is not None: + logging.info("Detected change to script file, submitting new task") + + prev_script_hash = script_hash + waiting_msg_printed = False + + task = createTask( + api_key, script, args.universe, args.place, args.place_version + ) + logging.info(f"Task created, path: {task['path']}") + + task = pollForTaskCompletion(api_key, task["path"]) + logging.info(f'Task is now in {task["state"]} state') + + handleLogs(task, args.log_output, api_key) + if task["state"] == "COMPLETE": + handleSuccess(task, args.output) + else: + handleFailure(task) + + if not args.continuous: + break diff --git a/.lune/open-cloud/upload_and_run_task.py b/.lune/open-cloud/upload_and_run_task.py new file mode 100644 index 00000000..264ceb1a --- /dev/null +++ b/.lune/open-cloud/upload_and_run_task.py @@ -0,0 +1,69 @@ +# upstream: https://github.com/Roblox/place-ci-cd-demo/blob/production/scripts/python/upload_and_run_task.py +import os +import sys +import urllib.request +import json + +from luau_execution_task import createTask, pollForTaskCompletion, getTaskLogs + +ROBLOX_API_KEY = os.environ["ROBLOX_API_KEY"] +ROBLOX_UNIVERSE_ID = os.environ["ROBLOX_UNIVERSE_ID"] +ROBLOX_PLACE_ID = os.environ["ROBLOX_PLACE_ID"] + + +def read_file(file_path): + with open(file_path, "rb") as file: + return file.read() + + +def upload_place(binary_path, universe_id, place_id, do_publish=False): + print("Uploading place to Roblox") + version_type = "Published" if do_publish else "Saved" + request_headers = { + "x-api-key": ROBLOX_API_KEY, + "Content-Type": "application/xml", + "Accept": "application/json", + } + + url = f"https://apis.roblox.com/universes/v1/{universe_id}/places/{place_id}/versions?versionType={version_type}" + + buffer = read_file(binary_path) + req = urllib.request.Request( + url, data=buffer, headers=request_headers, method="POST" + ) + + with urllib.request.urlopen(req) as response: + data = json.loads(response.read().decode("utf-8")) + place_version = data.get("versionNumber") + + return place_version + + +def run_luau_task(universe_id, place_id, place_version, script_file): + print("Executing Luau task") + script_contents = read_file(script_file).decode("utf8") + + task = createTask( + ROBLOX_API_KEY, script_contents, universe_id, place_id, place_version + ) + task = pollForTaskCompletion(ROBLOX_API_KEY, task["path"]) + logs = getTaskLogs(ROBLOX_API_KEY, task["path"]) + + print(logs) + + if task["state"] == "COMPLETE": + print("Luau task completed successfully") + exit(0) + else: + print("Luau task failed", file=sys.stderr) + exit(1) + + +if __name__ == "__main__": + universe_id = ROBLOX_UNIVERSE_ID + place_id = ROBLOX_PLACE_ID + binary_file = sys.argv[1] + script_file = sys.argv[2] + + place_version = upload_place(binary_file, universe_id, place_id) + run_luau_task(universe_id, place_id, place_version, script_file) diff --git a/.lune/publish-plugin.luau b/.lune/publish-plugin.luau new file mode 100644 index 00000000..46a4d86b --- /dev/null +++ b/.lune/publish-plugin.luau @@ -0,0 +1,51 @@ +local process = require("@lune/process") + +local parseArgs = require("./lib/parseArgs") +local run = require("./lib/run") +local clean = require("./lib/clean") +local compile = require("./lib/compile") +local project = require("../project") + +local args = parseArgs(process.args) + +local apiKey = assert(args.apiKey, "--apiKey must be supplied with a valid Open Cloud API key") +assert(typeof(apiKey) == "string", `bad value for apiKey (string expected, got {typeof(apiKey)})`) + +clean() +compile("dev") + +run("rojo", { "build", "tests.project.json", "-o", "tests.rbxl" }) + +local publishPluginAssetTask = "build/publish-plugin-asset.luau" + +do -- mini darklua for swapping out globals in the task with real values + run("cp", { ".lune/tasks/publish-plugin-asset.luau", publishPluginAssetTask }) + + local GLOBAL_SUBSTITUTIONS = { + ROBLOX_CREATOR_ID = project.ROBLOX_ASSET_CREATOR_ID, + ROBLOX_ASSET_ID = project.ROBLOX_PLUGIN_ASSET_ID_DEV, + } + + for global, value in GLOBAL_SUBSTITUTIONS do + run("sed", { "-i", "-e", `s/_G.{global}/{value}/g`, publishPluginAssetTask }) + end +end + +local output, success = run("python3", { + ".lune/open-cloud/upload_and_run_task.py", + "tests.rbxl", + publishPluginAssetTask, +}, { + env = { + ROBLOX_UNIVERSE_ID = project.ROBLOX_UNIVERSE_ID, + ROBLOX_PLACE_ID = project.ROBLOX_PLACE_ID, + ROBLOX_API_KEY = apiKey, + }, +}) + +run("rm", { "tests.rbxl" }) + +if not success then + print(output) + process.exit(1) +end diff --git a/.lune/tasks/publish-plugin-asset.luau b/.lune/tasks/publish-plugin-asset.luau new file mode 100644 index 00000000..3d59313f --- /dev/null +++ b/.lune/tasks/publish-plugin-asset.luau @@ -0,0 +1,40 @@ +local ReplicatedStorage = game:GetService("ReplicatedStorage") +local AssetService = game:GetService("AssetService") + +local CREATOR_ID: number = _G.ROBLOX_CREATOR_ID +assert(typeof(CREATOR_ID) == "number", "_G.ROBLOX_CREATOR_ID must be supplied") + +local ASSET_ID: number? = _G.ROBLOX_ASSET_ID + +local root = ReplicatedStorage:FindFirstChild("flipbook") +assert(root, "no plugin found for upload") + +local assetId: number? +local versionNumber = 1 + +if ASSET_ID == nil then + print("Creating plugin asset for the first time") + local _ + _, assetId = AssetService:CreateAssetAsync(root, Enum.AssetType.Plugin, { + Name = "Simple Plugin", + CreatorId = CREATOR_ID, + CreatorType = Enum.CreatorType.User, + }) +else + print(`Updating asset with ID {ASSET_ID}`) + assetId = ASSET_ID + local _ + _, versionNumber = AssetService:CreateAssetVersionAsync(root, Enum.AssetType.Plugin, assetId, { + CreatorId = CREATOR_ID, + CreatorType = Enum.CreatorType.User, + }) + print("Created new version", versionNumber) +end + +print("Plugin uploaded successfully! View it on the Creator Store:") +print(`https://create.roblox.com/store/asset/{assetId}`) + +return { + assetId = tostring(assetId), -- we make it a string because the Luau exec API serializes integers as floating points + versionNumber = tostring(versionNumber), +} diff --git a/project.luau b/project.luau new file mode 100644 index 00000000..6db7fd69 --- /dev/null +++ b/project.luau @@ -0,0 +1,8 @@ +return { + ROBLOX_UNIVERSE_ID = "6599100156", + ROBLOX_PLACE_ID = "123506190725771", + + ROBLOX_ASSET_CREATOR_ID = "1343930", + ROBLOX_PLUGIN_ASSET_ID_PROD = "8517129161", + ROBLOX_PLUGIN_ASSET_ID_DEV = "88523969718241", +} From 9e57e8575bccd127ccd9615654c3652802870b31 Mon Sep 17 00:00:00 2001 From: Marin Minnerly Date: Wed, 20 Nov 2024 15:25:25 -0800 Subject: [PATCH 2/6] Compile for prod, build to dev plugin --- .lune/build.luau | 12 ------------ .lune/lib/compile.luau | 12 ++++++++++++ .lune/publish-plugin.luau | 9 +++++++-- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/.lune/build.luau b/.lune/build.luau index cd917de7..2f827397 100644 --- a/.lune/build.luau +++ b/.lune/build.luau @@ -15,22 +15,10 @@ assert(target == "dev" or target == "prod", `bad value for target (must be one o local output = if args.output then args.output else `{getPluginsPath(process.os)}/{constants.PLUGIN_FILENAME}` assert(typeof(output) == "string", `bad value for output (string expected, got {typeof(output)})`) -local PRUNED_FILES = { - "*.spec.luau", - "*.story.luau", - "*.storybook.luau", -} - local function build() clean() compile(target) - if target == "prod" then - for _, pattern in PRUNED_FILES do - run("find", { constants.BUILD_PATH, "-type", "f", "-name", pattern, "-delete" }) - end - end - run("rojo", { "build", "-o", output }) end diff --git a/.lune/lib/compile.luau b/.lune/lib/compile.luau index 1b6db7f6..40577edd 100644 --- a/.lune/lib/compile.luau +++ b/.lune/lib/compile.luau @@ -4,6 +4,12 @@ local run = require("./run") type Target = "prod" | "dev" +local PRUNED_FILES = { + "*.spec.luau", + "*.story.luau", + "*.storybook.luau", +} + local function compile(target: Target) fs.writeDir(constants.BUILD_PATH) @@ -27,6 +33,12 @@ local function compile(target: Target) `{constants.BUILD_PATH}/Example`, }) end + + if target == "prod" then + for _, pattern in PRUNED_FILES do + run("find", { constants.BUILD_PATH, "-type", "f", "-name", pattern, "-delete" }) + end + end end return compile diff --git a/.lune/publish-plugin.luau b/.lune/publish-plugin.luau index 46a4d86b..004c13fd 100644 --- a/.lune/publish-plugin.luau +++ b/.lune/publish-plugin.luau @@ -8,11 +8,14 @@ local project = require("../project") local args = parseArgs(process.args) +local target = if args.target then args.target else "dev" +assert(target == "dev" or target == "prod", `bad value for target (must be one of "dev" or "prod", got "{target}")`) + local apiKey = assert(args.apiKey, "--apiKey must be supplied with a valid Open Cloud API key") assert(typeof(apiKey) == "string", `bad value for apiKey (string expected, got {typeof(apiKey)})`) clean() -compile("dev") +compile("prod") run("rojo", { "build", "tests.project.json", "-o", "tests.rbxl" }) @@ -23,7 +26,9 @@ do -- mini darklua for swapping out globals in the task with real values local GLOBAL_SUBSTITUTIONS = { ROBLOX_CREATOR_ID = project.ROBLOX_ASSET_CREATOR_ID, - ROBLOX_ASSET_ID = project.ROBLOX_PLUGIN_ASSET_ID_DEV, + ROBLOX_ASSET_ID = if target == "dev" + then project.ROBLOX_PLUGIN_ASSET_ID_DEV + else project.ROBLOX_PLUGIN_ASSET_ID_PROD, } for global, value in GLOBAL_SUBSTITUTIONS do From 4d709f8a79c2980a49bf6ad17b18d477b7af0902 Mon Sep 17 00:00:00 2001 From: Marin Minnerly Date: Wed, 20 Nov 2024 15:25:30 -0800 Subject: [PATCH 3/6] Attempt to deploy changes via workflow --- .github/workflows/release.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8398432a..e3fea61a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,3 +32,19 @@ jobs: files: ${{ env.MODEL_FILE }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + publish-plugin-nightly: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v3 + + - uses: Roblox/setup-foreman@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install packages + run: lune run wally-install + + - name: Run tests + run: lune run publish-plugin -- --apiKey ${{ secrets.ROBLOX_API_KEY }} From 66d1aeba88ee806b1af13e5aed6bc064b855f428 Mon Sep 17 00:00:00 2001 From: Marin Minnerly Date: Wed, 20 Nov 2024 15:28:35 -0800 Subject: [PATCH 4/6] Add a print to test the deployment --- src/init.server.luau | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/init.server.luau b/src/init.server.luau index 3a775379..82dfd50b 100644 --- a/src/init.server.luau +++ b/src/init.server.luau @@ -25,6 +25,8 @@ local widget = createWidget(plugin, PLUGIN_NAME) local root = ReactRoblox.createRoot(widget) local disconnectButton = createToggleButton(toolbar, widget) +print("This was deployed from GitHub!") + local loader = ModuleLoader.new() local app = React.createElement(ContextProviders, { From 30e04d7ae75be4fae8ff730c158ec688da1936c3 Mon Sep 17 00:00:00 2001 From: Marin Minnerly Date: Wed, 20 Nov 2024 15:29:52 -0800 Subject: [PATCH 5/6] Change step name --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e3fea61a..eaddc7ea 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,5 +46,5 @@ jobs: - name: Install packages run: lune run wally-install - - name: Run tests + - name: Publish to Creator Store run: lune run publish-plugin -- --apiKey ${{ secrets.ROBLOX_API_KEY }} From 32b2772f319b544e4c8c5a0e2ab87e8cd63c65e4 Mon Sep 17 00:00:00 2001 From: Marin Minnerly Date: Wed, 20 Nov 2024 16:15:33 -0800 Subject: [PATCH 6/6] Confirmed working. Now only publish on merge --- .github/workflows/release.yml | 1 + src/init.server.luau | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eaddc7ea..a2f08b50 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,6 +35,7 @@ jobs: publish-plugin-nightly: runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' timeout-minutes: 10 steps: - uses: actions/checkout@v3 diff --git a/src/init.server.luau b/src/init.server.luau index 82dfd50b..3a775379 100644 --- a/src/init.server.luau +++ b/src/init.server.luau @@ -25,8 +25,6 @@ local widget = createWidget(plugin, PLUGIN_NAME) local root = ReactRoblox.createRoot(widget) local disconnectButton = createToggleButton(toolbar, widget) -print("This was deployed from GitHub!") - local loader = ModuleLoader.new() local app = React.createElement(ContextProviders, {