Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Executable editing utils #1

Open
wants to merge 6 commits into
base: prolific
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .gitignore
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd prefer not to include the executables in this repo.

Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ build/
.DS_Store

# project specific
data/**
metadata/**
incomplete/**
reject/**
Expand All @@ -26,3 +25,8 @@ node_modules/
/playwright-report/
/blob-report/
/playwright/.cache/

# ignore all contents of data/ EXCEPT data/base_executables & contents of base_executables
data/**
!data/base_executables
!data/base_executables/**
7 changes: 4 additions & 3 deletions app/config.py
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you think you could store the task version (maybe just the commit hash) in the executables themselves? Then we could include that version string in the app.ini file and have version checking and executable downloading from OSF happen in here.

Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@
if not os.path.isdir(task_badupload_dir): os.makedirs(task_badupload_dir)
dl_dir = os.path.join(data_dir, 'download')
if not os.path.isdir(dl_dir): os.makedirs(dl_dir)
exe_dir = os.path.join(data_dir, 'exe')
if not os.path.isdir(exe_dir): os.makedirs(exe_dir)
base_exe_dir = os.path.join(data_dir, 'base_executables')
if not os.path.isdir(base_exe_dir): os.makedirs(base_exe_dir)
survey_dir = os.path.join(data_dir, 'survey')
if not os.path.isdir(survey_dir): os.makedirs(survey_dir)
survey_incomplete_dir = os.path.join(survey_dir, 'incomplete')
Expand Down Expand Up @@ -65,7 +65,8 @@
s_reject=survey_reject_dir,
s_complete=survey_complete_dir,
download=dl_dir,
exe=dl_dir,
base_exe=os.path.join(base_exe_dir, 'SUPREME.exe'),
base_app=os.path.join(base_exe_dir, 'SUPREME.app'),
disallowed_agents=json.loads(cfg['FLASK']['DISALLOWED_AGENTS']),
allowed_agents=json.loads(cfg['FLASK']['ALLOWED_AGENTS']),
blocks=json.loads(cfg['SUPREME']['BLOCKS']),
Expand Down
6 changes: 3 additions & 3 deletions app/taskstart.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from .io import write_metadata, initialize_taskdata
from .config import CFG
from .routing import routing
from .utils import make_download
from .utils import edit_exe_worker_id, edit_app_worker_id
from hashlib import blake2b


Expand All @@ -29,8 +29,8 @@ def taskstart():
supreme_subid = current_app.config['SUPREME_serializer'].dumps(h_workerId)
win_dlpath = os.path.join(CFG['download'], 'win_' + str(session['subId']) + '.exe')
mac_dlpath = os.path.join(CFG['download'], 'mac_' + str(session['subId']) + '.app')
make_download(supreme_subid, win_dlpath, 'windows')
make_download(supreme_subid, mac_dlpath, 'mac')
edit_exe_worker_id(exe_file_path=CFG['base_exe'], new_worker_id=supreme_subid, output_file_path=win_dlpath)
edit_app_worker_id(app_path=CFG['base_app'], new_worker_id=supreme_subid, output_app_path=mac_dlpath)
session['dlready'] = True
write_metadata(session, ['dlready'], 'a')
initialize_taskdata(session)
Expand Down
147 changes: 140 additions & 7 deletions app/utils.py
Original file line number Diff line number Diff line change
@@ -1,32 +1,36 @@
import random
import string
import copy
import logging
import os
import plistlib
import shutil
from typing import Optional
from pathlib import Path
from pefile import PE, DIRECTORY_ENTRY


def gen_code(N):
"""Generate random completion code."""
return ''.join(random.choices(string.ascii_lowercase + string.digits, k=N))


def make_download(sid, dlpath, platform):
pass


def pseudorandomize(inblocks, nreps, shuffle_blocks=True, nested_output=False):
"""
pseudorandomize the input blocks so that each task occurs once before
any task is repeated and no tasks occur back to back. Will trigger an
infinite loop if the number of blocks it too small to generate enough unique
orders to satisfy the shuffle blocks condition for the number of repititions
orders to satisfy the shuffle blocks condition for the number of repetitions
requested.

Parameters
==========
inblocks : list of strings or list of list of strings
list of tasks to randomize
nreps : int
number of repititions of each task
number of repetitions of each task
shuffle blocks : bool
Should task order be shuffeled everytime they're repeated
Should task order be shuffled every time they're repeated
nested_output : bool
Should the output be nested (list of lists)

Expand Down Expand Up @@ -54,3 +58,132 @@ def pseudorandomize(inblocks, nreps, shuffle_blocks=True, nested_output=False):
for task in task_block:
flat_blocks.append(task)
return flat_blocks


logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s')


def edit_app_worker_id(app_path: str, new_worker_id: str, output_app_path: Optional[str] = None) -> None:
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think the output_app_path should be optional in the version of this code included in the backend. I'd like to avoid the possibility of the base apps getting accidentally overwritten.

"""
Modifies the 'WorkerID' field in the Info.plist of a macOS .app bundle.

Args:
app_path (str): Path to the original .app bundle whose 'WorkerID' will be modified.
new_worker_id (str): New 'WorkerID' value to replace the existing one.
output_app_path (Optional[str]): Path to save the modified .app bundle. If not provided,
the original bundle will be overwritten.

Raises:
FileNotFoundError: If the specified .app bundle or Info.plist file does not exist.
ValueError: If the Info.plist file cannot be loaded or parsed.
"""

# Construct the path to the Info.plist file inside the original .app bundle
plist_path = Path(app_path) / 'Contents' / 'Info.plist'

# Ensure the .app bundle and Info.plist exist
if not plist_path.exists():
raise FileNotFoundError(f"The file {plist_path} does not exist.")

try:
# If an output path is specified, copy the original .app bundle to the new location
if output_app_path:
output_path = Path(output_app_path)
if output_path.exists():
raise FileExistsError(
f"The output path {output_app_path} already exists.")
shutil.copytree(app_path, output_app_path)
# Update plist path to the new location
plist_path = output_path / 'Contents' / 'Info.plist'
else:
logging.info(
"No output path specified. The original .app bundle will be modified.")

# Load the plist file
with plist_path.open('rb') as plist_file:
plist_data = plistlib.load(plist_file)

# Log current WorkerID, if present
current_worker_id = plist_data.get('WorkerID', None)
if current_worker_id:
logging.info(f"Current WorkerID: {current_worker_id}")

# Update the WorkerID
plist_data['WorkerID'] = new_worker_id

# Write the updated plist back
with plist_path.open('wb') as plist_file:
plistlib.dump(plist_data, plist_file)

logging.info(f"Successfully updated WorkerID to {new_worker_id}")

except Exception as e:
raise ValueError(f"Failed to modify the plist file: {e}")


def edit_exe_worker_id(exe_file_path: str, new_worker_id: str, output_file_path: Optional[str] = None) -> None:
"""
Modifies the 'WorkerID' field in the version information of an executable.

Args:
exe_file_path (str): Path to the executable whose 'WorkerID' will be modified.
new_worker_id (str): New 'WorkerID' value to replace the existing one.
**Must** be less than or equal to the length of the current 'WorkerID' for a successful update.
output_file_path (str): **Optional**. Path where the modified executable will be saved.
If not provided, the original executable will be overwritten.

Raises:
FileNotFoundError: If the specified executable does not exist.
ValueError: If the PE file cannot be loaded or parsed.
"""

if not os.path.exists(exe_file_path):
raise FileNotFoundError(f"The file {exe_file_path} does not exist.")

output_file_path = Path(output_file_path or exe_file_path)

try:
pe: PE = PE(exe_file_path, fast_load=True)
pe.parse_data_directories(
directories=[DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_RESOURCE']])
except Exception as e:
raise ValueError(f"Failed to load or parse the PE file: {e}")

# Access the WorkerID and update it if found
if hasattr(pe, 'FileInfo'):
for file_info in pe.FileInfo:
for info in file_info:
if info.name == 'StringFileInfo':
version_info_dict: dict = info.StringTable[0].entries
worker_id_bytes: bytes = version_info_dict.get(
b'WorkerID', None)

if worker_id_bytes:
logging.info(
f"Found WorkerID: {worker_id_bytes.decode('utf-8')}")

# Calculate the original size in bytes and the new value size
original_size: int = len(worker_id_bytes)
new_worker_id_bytes: bytes = new_worker_id.encode(
'utf-8')
new_size: int = len(new_worker_id_bytes)

if new_size <= original_size:
# Create the new value padded with null bytes up to the original size
padded_new_worker_id: bytes = new_worker_id_bytes + \
b'\x00' * (original_size - new_size)

# Update the value with the padded version
version_info_dict[b"WorkerID"] = padded_new_worker_id

# Write the updated attribute to the output file
pe.write(output_file_path)
logging.info(
f"Successfully updated WorkerID to {new_worker_id}")
else:
logging.error(
f"Error: New value '{new_worker_id}' is larger than the existing space of {original_size} bytes.")
return

logging.error(f"Error: WorkerID not found in {exe_file_path}")
33 changes: 33 additions & 0 deletions data/base_executables/SUPREME.app/Contents/Info.plist
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDisplayName</key>
<string>SUPREME</string>
<key>CFBundleExecutable</key>
<string>SUPREME</string>
<key>CFBundleIconFile</key>
<string>icon-windowed.icns</string>
<key>CFBundleIdentifier</key>
<string>uva.compmem.supreme</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>SUPREME</string>
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleShortVersionString</key>
<string>1.0.0</string>
<key>CFBundleVersion</key>
<string>1.0.0</string>
<key>LSArchitecturePriority</key>
<array>
<string>x86_64</string>
<string>arm64</string>
</array>
<key>NSHighResolutionCapable</key>
<string>True</string>
<key>WorkerID</key>
<string>"------------------------".---------------------------</string>
</dict>
</plist>
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>files</key>
<dict>
<key>Resources/icon-windowed.icns</key>
<data>
eEHOuYpZLB0vKGVIWGZOh5rH8+o=
</data>
</dict>
<key>files2</key>
<dict>
<key>Resources/icon-windowed.icns</key>
<dict>
<key>hash2</key>
<data>
uQo7VuWRab4Phv4EEGmfQsyqFqDIXZgO8OtgaAMvCzY=
</data>
</dict>
</dict>
<key>rules</key>
<dict>
<key>^Resources/</key>
<true/>
<key>^Resources/.*\.lproj/</key>
<dict>
<key>optional</key>
<true/>
<key>weight</key>
<real>1000</real>
</dict>
<key>^Resources/.*\.lproj/locversion.plist$</key>
<dict>
<key>omit</key>
<true/>
<key>weight</key>
<real>1100</real>
</dict>
<key>^Resources/Base\.lproj/</key>
<dict>
<key>weight</key>
<real>1010</real>
</dict>
<key>^version.plist$</key>
<true/>
</dict>
<key>rules2</key>
<dict>
<key>.*\.dSYM($|/)</key>
<dict>
<key>weight</key>
<real>11</real>
</dict>
<key>^(.*/)?\.DS_Store$</key>
<dict>
<key>omit</key>
<true/>
<key>weight</key>
<real>2000</real>
</dict>
<key>^(Frameworks|SharedFrameworks|PlugIns|Plug-ins|XPCServices|Helpers|MacOS|Library/(Automator|Spotlight|LoginItems))/</key>
<dict>
<key>nested</key>
<true/>
<key>weight</key>
<real>10</real>
</dict>
<key>^.*</key>
<true/>
<key>^Info\.plist$</key>
<dict>
<key>omit</key>
<true/>
<key>weight</key>
<real>20</real>
</dict>
<key>^PkgInfo$</key>
<dict>
<key>omit</key>
<true/>
<key>weight</key>
<real>20</real>
</dict>
<key>^Resources/</key>
<dict>
<key>weight</key>
<real>20</real>
</dict>
<key>^Resources/.*\.lproj/</key>
<dict>
<key>optional</key>
<true/>
<key>weight</key>
<real>1000</real>
</dict>
<key>^Resources/.*\.lproj/locversion.plist$</key>
<dict>
<key>omit</key>
<true/>
<key>weight</key>
<real>1100</real>
</dict>
<key>^Resources/Base\.lproj/</key>
<dict>
<key>weight</key>
<real>1010</real>
</dict>
<key>^[^/]+$</key>
<dict>
<key>nested</key>
<true/>
<key>weight</key>
<real>10</real>
</dict>
<key>^embedded\.provisionprofile$</key>
<dict>
<key>weight</key>
<real>20</real>
</dict>
<key>^version\.plist$</key>
<dict>
<key>weight</key>
<real>20</real>
</dict>
</dict>
</dict>
</plist>
Binary file added data/base_executables/SUPREME.exe
Binary file not shown.
Loading