Test Python version >= 3.10 in CI and fix tests on Python 3.12 (#2486)

* Bump tested python version to >= 3.10

Some workflows were using 3.8 still. Start testing on 3.12 too.

* Fix installing rpyc in CI on ubuntu 24.04

`ubuntu-latest` now points to 24.04 which requires the --break-system-packages dance.

* Test Python 2 on Ubuntu 22.04

Ubuntu 24.04 dropped the python2.7 packages.

* Test on android-34

* Fix ARM binutils disassembly output test expectation

It used to print "; 0x4" but changed to "@ 0x4" in some version.

* Skip QEMU LD_PREFIX path test

Ubuntu 24.04 seemed to have switched the qemu-user --help output to show `/usr/gnemul/qemu-$ARCH` instead of `/etc/qemu-binfmt`. Ignore the actual path in the test.

* Fix x86 ascii shellcode encoder test

The alphabet was using `\` escape sequences badly.

<stdin>:1: SyntaxWarning: invalid escape sequence '\]'

* Fix mips xor encoder unaligned memory access

qemu throws a SIGBUS error when the shellcode tries to access unaligned memory since some version. Align the "stack" properly.

* Fix util.lists.partition test output on Python 3.12

The __repr__ output of OrderedDict was changed to look like ordinary {} dicts.

* Fix safeeval test on Python 3.12

There is a 151 RESUME opcode in Python 3.12 now.

* Fix registering commandline subparsers multiple times for Python 3.11

Python 3.11 added a sanity check to argparse to prevent registering the same subparser multiple times.

argparse.ArgumentError: argument command: conflicting subparser: cyclic

Avoid importing the command twice.
`python -m pwnlib.commandline.cyclic` failed on Python 3.11 triggered in CI when running the commandline tools while collecting coverage.

* Fix pip cache in CI

.github/workflows/android.yml

@@ -6,24 +6,21 @@ jobs:
   android-test:
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.10']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
     - uses: actions/checkout@v4
 
-    - name: Cache for pip
-      uses: actions/cache@v4
-      id: cache-pip
-      with:
-        path: ~/.cache/pip
-        key: ${{ matrix.os }}-cache-pip
-
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          **/pyproject.toml
+          **/requirements*.txt
 
     - name: Install Linux dependencies
       run: |
@@ -35,6 +32,19 @@ jobs:
           binutils-arm-linux-gnueabihf \
           libc6-dbg
 
+    - name: Cache for avd
+      uses: actions/cache@v4
+      id: cache-avd
+      with:
+        path: |
+          ~/.android
+          /usr/local/lib/android/sdk/emulator
+          /usr/local/lib/android/sdk/platform-tools
+          /usr/local/lib/android/sdk/system-images
+        key: ${{ matrix.os }}-cache-avd-${{ hashFiles('travis/setup_avd*.sh') }}
+        restore-keys: |
+          ${{ matrix.os }}-cache-avd-
+
     - name: Install Android AVD
       run: |
         sudo usermod -aG kvm $USER

.github/workflows/ci.yml

@@ -5,8 +5,11 @@ jobs:
   test:
     strategy:
       matrix:
-        python_version: ['2.7', '3.10']
+        python_version: ['3.10', '3.12']
         os: [ubuntu-latest]
+        include:
+          - python_version: '2.7'
+            os: ubuntu-22.04
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
@@ -21,23 +24,30 @@ jobs:
 
     - name: Install RPyC for gdb
       run: |
-        # The version packaged in python3-rpyc is too old on Ubuntu 22.04
+        # The version packaged in python3-rpyc is too old on Ubuntu 24.04
+        # We use ^6.0 from pip.
         sudo apt-get update && sudo apt-get install -y python3-pip gdb gdbserver
-        /usr/bin/python -m pip install rpyc
+        /usr/bin/python -m pip install --break-system-packages rpyc || /usr/bin/python -m pip install rpyc
         gdb --batch --quiet --nx --nh --ex 'py import rpyc; print(rpyc.version.version)'
 
     - name: Cache for pip
       uses: actions/cache@v4
+      if: matrix.python_version == '2.7'
       id: cache-pip
       with:
         path: ~/.cache/pip
-        key: ${{ matrix.os }}-cache-pip
+        key: ${{ matrix.os }}-${{ matrix.python_version }}-cache-pip-${{ hashFiles('**/pyproject.toml', '**/requirements*.txt') }}
+        restore-keys: ${{ matrix.os }}-${{ matrix.python_version }}-cache-pip-
 
     - name: Set up Python ${{ matrix.python_version }}
       if: matrix.python_version != '2.7'
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python_version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          **/pyproject.toml
+          **/requirements*.txt
 
     - name: Set up Python 2.7
       if: matrix.python_version == '2.7'
@@ -195,15 +205,17 @@ jobs:
         python -m build
 
     - uses: actions/upload-artifact@v4
-      if: matrix.python_version != '2.7'
+      if: matrix.python_version == '3.10'
       with:
         name: packages
         path: dist/
+        include-hidden-files: true
 
     - uses: actions/upload-artifact@v4
       with:
         name: coverage-${{ matrix.python_version }}
         path: .coverage*
+        include-hidden-files: true
 
 
   upload-coverage:
@@ -221,7 +233,7 @@ jobs:
 
     - name: Install coveralls
       run: |
-        pip install tomli coveralls
+        pip install --break-system-packages tomli coveralls
 
     - name: Upload coverage to coveralls.io
       run: |

.github/workflows/lint.yml

@@ -5,23 +5,21 @@ jobs:
   lint:
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.10']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
     - uses: actions/checkout@v4
-    - name: Cache for pip
-      uses: actions/cache@v4
-      id: cache-pip
-      with:
-        path: ~/.cache/pip
-        key: ${{ matrix.os }}-cache-pip
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          **/pyproject.toml
+          **/requirements*.txt
 
     - name: Critical lint
       run: |

.github/workflows/pylint.yml

@@ -5,23 +5,21 @@ jobs:
   build:
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.10']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
     - uses: actions/checkout@v4
-    - name: Cache for pip
-      uses: actions/cache@v4
-      id: cache-pip
-      with:
-        path: ~/.cache/pip
-        key: ${{ matrix.os }}-cache-pip
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          **/pyproject.toml
+          **/requirements*.txt
 
     - name: PyLint
       run: |

pwnlib/adb/adb.py

@@ -123,7 +123,7 @@ def current_device(any=False):
 
         >>> device = adb.current_device(any=True)
         >>> device  # doctest: +ELLIPSIS
-        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_...phone_...', model='...', device='generic...')
+        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_...phone..._...', model='...', device='...')
         >>> device.port
         'emulator'
     """
@@ -259,7 +259,7 @@ class AdbDevice(Device):
         >>> device.os
         'android'
         >>> device.product  # doctest: +ELLIPSIS
-        'sdk_...phone_...'
+        'sdk_...phone..._...'
         >>> device.serial
         'emulator-5554'
     """
@@ -880,7 +880,7 @@ def which(name, all = False, *a, **kw):
         >>> adb.which('sh')
         '/system/bin/sh'
         >>> adb.which('sh', all=True)
-        ['/system/bin/sh']
+        ['/system/bin/sh', '/vendor/bin/sh']
 
         >>> adb.which('foobar') is None
         True
@@ -988,7 +988,7 @@ def proc_exe(pid):
        :skipif: skip_android
 
         >>> adb.proc_exe(1)
-        b'/init'
+        b'/system/bin/init'
     """
     with context.quiet:
         io  = process(['realpath','/proc/%d/exe' % pid])
@@ -1365,7 +1365,7 @@ def compile(source):
         >>> filename = adb.compile(temp)
         >>> sent = adb.push(filename, "/data/local/tmp")
         >>> adb.process(sent).recvall() # doctest: +ELLIPSIS
-        b'... /system/lib64/libc.so\n...'
+        b'... /system/lib64/libc++.so\n...'
     """
 
     ndk_build = misc.which('ndk-build')

pwnlib/asm.py

@@ -855,8 +855,8 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
            0:   b8 17 00 00 00          mov    eax, 0x17
         >>> print(disasm(unhex('48c7c017000000'), arch = 'amd64'))
            0:   48 c7 c0 17 00 00 00    mov    rax, 0x17
-        >>> print(disasm(unhex('04001fe552009000'), arch = 'arm'))
-           0:   e51f0004        ldr     r0, [pc, #-4]   ; 0x4
+        >>> print(disasm(unhex('04001fe552009000'), arch = 'arm'))  # doctest: +ELLIPSIS
+           0:   e51f0004        ldr     r0, [pc, #-4]   ...
            4:   00900052        addseq  r0, r0, r2, asr r0
         >>> print(disasm(unhex('4ff00500'), arch = 'thumb', bits=32))
            0:   f04f 0005       mov.w   r0, #5

pwnlib/commandline/asm.py

@@ -137,4 +137,4 @@ def main(args):
         args.output.write(b'\n')
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/checksec.py

@@ -38,4 +38,4 @@ def main(args):
         e = ELF(f.name)
 
 if __name__ == '__main__':
-    common.main(__file__)
+    common.main(__file__, main)

pwnlib/commandline/common.py

@@ -25,8 +25,18 @@ def context_arg(arg):
                                  prog='pwn')
 parser_commands = parser.add_subparsers(dest='command')
 
-def main(file=sys.argv[0]):
-    import pwnlib.commandline.main
+def main(file=sys.argv[0], command_main=None):
     name = os.path.splitext(os.path.basename(file))[0]
+    if command_main is None:
+        import importlib
+        command_main = importlib.import_module('pwnlib.commandline.%s' % name).main
     sys.argv.insert(1, name)
-    pwnlib.commandline.main.main()
+    entrypoint({name: command_main})
+
+def entrypoint(commands):
+    if len(sys.argv) < 2:
+        parser.print_usage()
+        sys.exit()
+    args = parser.parse_args()
+    with context.local(log_console = sys.stderr):
+        commands[args.command](args)

pwnlib/commandline/constgrep.py

@@ -133,4 +133,4 @@ def main(args):
                 print('(%s) == %s' % (' | '.join(k for v, k in good), args.constant))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/cyclic.py

@@ -107,4 +107,4 @@ def main(args):
             out.write(b'\n')
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/debug.py

@@ -102,4 +102,4 @@ def main(args):
         gdb.debug(target, gdbscript=gdbscript, sysroot=args.sysroot).interactive()
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/disablenx.py

@@ -24,4 +24,4 @@ def main(args):
         ELF(e.path)
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/disasm.py

@@ -110,4 +110,4 @@ def main(args):
     print(disasm(dat, vma=safeeval.const(args.address)))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/elfdiff.py

@@ -59,4 +59,4 @@ def main(a):
     print(diff(x, y))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/elfpatch.py

@@ -34,4 +34,4 @@ def main(a):
     getattr(sys.stdout, 'buffer', sys.stdout).write(elf.get_data())
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/errno.py

@@ -46,4 +46,4 @@ def main(args):
   print(os.strerror(value))
 
 if __name__ == '__main__':
-    common.main(__file__)
+    common.main(__file__, main)

pwnlib/commandline/hex.py

@@ -50,4 +50,4 @@ def main(args):
     print(encoded)
 
 if __name__ == '__main__':
-    common.main(__file__)
+    common.main(__file__, main)

pwnlib/commandline/libcdb.py

@@ -248,4 +248,4 @@ def main(args):
                     log.indented('%25s = %#x', symbol, translate_offset(exe.symbols[symbol], args, exe))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/main.py

@@ -1,7 +1,5 @@
 from __future__ import absolute_import
 
-import sys
-
 from pwnlib.commandline import asm
 from pwnlib.commandline import checksec
 from pwnlib.commandline import common
@@ -23,8 +21,7 @@
 from pwnlib.commandline import unhex
 from pwnlib.commandline import update
 from pwnlib.commandline import version
-from pwnlib.commandline.common import parser
-from pwnlib.context import context
+from pwnlib.commandline.common import parser as parser
 
 commands = {
     'asm': asm.main,
@@ -50,12 +47,7 @@
 }
 
 def main():
-    if len(sys.argv) < 2:
-        parser.print_usage()
-        sys.exit()
-    args = parser.parse_args()
-    with context.local(log_console = sys.stderr):
-        commands[args.command](args)
+    common.entrypoint(commands)
 
 if __name__ == '__main__':
     main()

pwnlib/commandline/phd.py

@@ -109,4 +109,4 @@ def main(args):
         pass
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/pwnstrip.py

@@ -53,4 +53,4 @@ def main(args):
     args.output.write(result)
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/scramble.py

@@ -110,4 +110,4 @@ def main(args):
 
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/shellcraft.py

@@ -352,4 +352,4 @@ def main(args):
     args.out.write(code)
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/template.py

@@ -122,5 +122,5 @@ def main(args):
         except OSError: pass
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)