diff --git a/.gitignore b/.gitignore index 28a44f4f..63bbbe1e 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ lib64/ parts/ sdist/ var/ +*.dist-info/ *.egg-info/ .installed.cfg *.egg @@ -33,3 +34,21 @@ nosetests.xml coverage.xml *,cover +# Sphinx documentation +docs/_build/ + +############# +## Custom +############# + +# virtualenvs +venv/ +ENV/ +.venv* +.venv/* + +# embedded libs +ext_libs/bin +ext_libs/docs +ext_libs/*/contrib/ +ext_libs/*/tests/ diff --git a/docs/development/update_dependencies.md b/docs/development/update_dependencies.md new file mode 100644 index 00000000..7a644b07 --- /dev/null +++ b/docs/development/update_dependencies.md @@ -0,0 +1,24 @@ +# Dependencies upgrade workflow + +This plugin has external dependencies: + +- dulwich +- giturlparse +- pathvalidate + +Because it's still hard to install Python 3rd party packages from an index (for example ), especially on Windows or Mac systems (or even on Linux if we want to do it properly in a virtual environment), those required packages are stored in the `ext_libs` folder. + +## Upgrade workflow + +Manage versions in the `requirements/embedded.txt` file, then: + +```bash +python -m pip install --no-deps -U -r requirements/embedded.txt -t ext_libs +``` + +Note: even if `dulwich` depends on `certifi` and `urllib3`, we specifally install them since they are already included with QGIS. + +## Related links + +- +- diff --git a/ext_libs/dulwich/__init__.py b/ext_libs/dulwich/__init__.py index 526be8aa..b6767615 100644 --- a/ext_libs/dulwich/__init__.py +++ b/ext_libs/dulwich/__init__.py @@ -22,4 +22,4 @@ """Python implementation of the Git file formats and protocols.""" -__version__ = (0, 19, 15) +__version__ = (0, 20, 6) diff --git a/ext_libs/dulwich/_diff_tree.c b/ext_libs/dulwich/_diff_tree.c index 068b4f63..88d17ed9 100644 --- a/ext_libs/dulwich/_diff_tree.c +++ b/ext_libs/dulwich/_diff_tree.c @@ -26,25 +26,6 @@ typedef unsigned short mode_t; #endif -#if PY_MAJOR_VERSION < 3 -typedef long Py_hash_t; -#endif - -#if PY_MAJOR_VERSION >= 3 -#define PyInt_FromLong PyLong_FromLong -#define PyInt_AsLong PyLong_AsLong -#define PyInt_AS_LONG PyLong_AS_LONG -#define PyString_AS_STRING PyBytes_AS_STRING -#define PyString_AsStringAndSize PyBytes_AsStringAndSize -#define PyString_Check PyBytes_Check -#define PyString_CheckExact PyBytes_CheckExact -#define PyString_FromStringAndSize PyBytes_FromStringAndSize -#define PyString_FromString PyBytes_FromString -#define PyString_GET_SIZE PyBytes_GET_SIZE -#define PyString_Size PyBytes_Size -#define _PyString_Join _PyBytes_Join -#endif - static PyObject *tree_entry_cls = NULL, *null_entry = NULL, *defaultdict_cls = NULL, *int_cls = NULL; static int block_size; @@ -118,7 +99,7 @@ static PyObject **tree_entries(char *path, Py_ssize_t path_len, PyObject *tree, if (!sha) goto error; name = PyTuple_GET_ITEM(old_entry, 0); - name_len = PyString_Size(name); + name_len = PyBytes_Size(name); if (PyErr_Occurred()) goto error; @@ -133,18 +114,13 @@ static PyObject **tree_entries(char *path, Py_ssize_t path_len, PyObject *tree, if (path_len) { memcpy(new_path, path, path_len); new_path[path_len] = '/'; - memcpy(new_path + path_len + 1, PyString_AS_STRING(name), name_len); + memcpy(new_path + path_len + 1, PyBytes_AS_STRING(name), name_len); } else { - memcpy(new_path, PyString_AS_STRING(name), name_len); + memcpy(new_path, PyBytes_AS_STRING(name), name_len); } -#if PY_MAJOR_VERSION >= 3 result[i] = PyObject_CallFunction(tree_entry_cls, "y#OO", new_path, new_path_len, PyTuple_GET_ITEM(old_entry, 1), sha); -#else - result[i] = PyObject_CallFunction(tree_entry_cls, "s#OO", new_path, - new_path_len, PyTuple_GET_ITEM(old_entry, 1), sha); -#endif PyMem_Free(new_path); if (!result[i]) { goto error; @@ -172,7 +148,7 @@ static int entry_path_cmp(PyObject *entry1, PyObject *entry2) if (!path1) goto done; - if (!PyString_Check(path1)) { + if (!PyBytes_Check(path1)) { PyErr_SetString(PyExc_TypeError, "path is not a (byte)string"); goto done; } @@ -181,12 +157,12 @@ static int entry_path_cmp(PyObject *entry1, PyObject *entry2) if (!path2) goto done; - if (!PyString_Check(path2)) { + if (!PyBytes_Check(path2)) { PyErr_SetString(PyExc_TypeError, "path is not a (byte)string"); goto done; } - result = strcmp(PyString_AS_STRING(path1), PyString_AS_STRING(path2)); + result = strcmp(PyBytes_AS_STRING(path1), PyBytes_AS_STRING(path2)); done: Py_XDECREF(path1); @@ -202,11 +178,7 @@ static PyObject *py_merge_entries(PyObject *self, PyObject *args) char *path_str; int cmp; -#if PY_MAJOR_VERSION >= 3 if (!PyArg_ParseTuple(args, "y#OO", &path_str, &path_len, &tree1, &tree2)) -#else - if (!PyArg_ParseTuple(args, "s#OO", &path_str, &path_len, &tree1, &tree2)) -#endif return NULL; entries1 = tree_entries(path_str, path_len, tree1, &n1); @@ -270,6 +242,11 @@ static PyObject *py_merge_entries(PyObject *self, PyObject *args) return result; } +/* Not all environments define S_ISDIR */ +#if !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR) +#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) +#endif + static PyObject *py_is_tree(PyObject *self, PyObject *args) { PyObject *entry, *mode, *result; @@ -286,7 +263,7 @@ static PyObject *py_is_tree(PyObject *self, PyObject *args) result = Py_False; Py_INCREF(result); } else { - lmode = PyInt_AsLong(mode); + lmode = PyLong_AsLong(mode); if (lmode == -1 && PyErr_Occurred()) { Py_DECREF(mode); return NULL; @@ -305,13 +282,13 @@ static Py_hash_t add_hash(PyObject *get, PyObject *set, char *str, int n) /* It would be nice to hash without copying str into a PyString, but that * isn't exposed by the API. */ - str_obj = PyString_FromStringAndSize(str, n); + str_obj = PyBytes_FromStringAndSize(str, n); if (!str_obj) goto error; hash = PyObject_Hash(str_obj); if (hash == -1) goto error; - hash_obj = PyInt_FromLong(hash); + hash_obj = PyLong_FromLong(hash); if (!hash_obj) goto error; @@ -319,7 +296,7 @@ static Py_hash_t add_hash(PyObject *get, PyObject *set, char *str, int n) if (!value) goto error; set_value = PyObject_CallFunction(set, "(Ol)", hash_obj, - PyInt_AS_LONG(value) + n); + PyLong_AS_LONG(value) + n); if (!set_value) goto error; @@ -372,11 +349,11 @@ static PyObject *py_count_blocks(PyObject *self, PyObject *args) for (i = 0; i < num_chunks; i++) { chunk = PyList_GET_ITEM(chunks, i); - if (!PyString_Check(chunk)) { + if (!PyBytes_Check(chunk)) { PyErr_SetString(PyExc_TypeError, "chunk is not a string"); goto error; } - if (PyString_AsStringAndSize(chunk, &chunk_str, &chunk_len) == -1) + if (PyBytes_AsStringAndSize(chunk, &chunk_str, &chunk_len) == -1) goto error; for (j = 0; j < chunk_len; j++) { @@ -420,7 +397,6 @@ moduleinit(void) PyObject *m, *objects_mod = NULL, *diff_tree_mod = NULL; PyObject *block_size_obj = NULL; -#if PY_MAJOR_VERSION >= 3 static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "_diff_tree", /* m_name */ @@ -433,9 +409,6 @@ moduleinit(void) NULL, /* m_free */ }; m = PyModule_Create(&moduledef); -#else - m = Py_InitModule("_diff_tree", py_diff_tree_methods); -#endif if (!m) goto error; @@ -459,7 +432,7 @@ moduleinit(void) block_size_obj = PyObject_GetAttrString(diff_tree_mod, "_BLOCK_SIZE"); if (!block_size_obj) goto error; - block_size = (int)PyInt_AsLong(block_size_obj); + block_size = (int)PyLong_AsLong(block_size_obj); if (PyErr_Occurred()) goto error; @@ -490,16 +463,8 @@ moduleinit(void) return NULL; } -#if PY_MAJOR_VERSION >= 3 PyMODINIT_FUNC PyInit__diff_tree(void) { return moduleinit(); } -#else -PyMODINIT_FUNC -init_diff_tree(void) -{ - moduleinit(); -} -#endif diff --git a/ext_libs/dulwich/_objects.c b/ext_libs/dulwich/_objects.c index 417e189d..34635e82 100644 --- a/ext_libs/dulwich/_objects.c +++ b/ext_libs/dulwich/_objects.c @@ -23,15 +23,6 @@ #include #include -#if PY_MAJOR_VERSION >= 3 -#define PyInt_Check(obj) 0 -#define PyInt_CheckExact(obj) 0 -#define PyInt_AsLong PyLong_AsLong -#define PyString_AS_STRING PyBytes_AS_STRING -#define PyString_Check PyBytes_Check -#define PyString_FromStringAndSize PyBytes_FromStringAndSize -#endif - #if defined(__MINGW32_VERSION) || defined(__APPLE__) size_t rep_strnlen(char *text, size_t maxlen); size_t rep_strnlen(char *text, size_t maxlen) @@ -56,7 +47,7 @@ static PyObject *sha_to_pyhex(const unsigned char *sha) hexsha[i*2+1] = bytehex(sha[i] & 0x0F); } - return PyString_FromStringAndSize(hexsha, 40); + return PyBytes_FromStringAndSize(hexsha, 40); } static PyObject *py_parse_tree(PyObject *self, PyObject *args, PyObject *kw) @@ -67,13 +58,8 @@ static PyObject *py_parse_tree(PyObject *self, PyObject *args, PyObject *kw) PyObject *ret, *item, *name, *sha, *py_strict = NULL; static char *kwlist[] = {"text", "strict", NULL}; -#if PY_MAJOR_VERSION >= 3 if (!PyArg_ParseTupleAndKeywords(args, kw, "y#|O", kwlist, &text, &len, &py_strict)) -#else - if (!PyArg_ParseTupleAndKeywords(args, kw, "s#|O", kwlist, - &text, &len, &py_strict)) -#endif return NULL; strict = py_strict ? PyObject_IsTrue(py_strict) : 0; /* TODO: currently this returns a list; if memory usage is a concern, @@ -100,7 +86,7 @@ static PyObject *py_parse_tree(PyObject *self, PyObject *args, PyObject *kw) } text++; namelen = strnlen(text, len - (text - start)); - name = PyString_FromStringAndSize(text, namelen); + name = PyBytes_FromStringAndSize(text, namelen); if (name == NULL) { Py_DECREF(ret); return NULL; @@ -141,6 +127,11 @@ struct tree_item { PyObject *tuple; }; +/* Not all environments define S_ISDIR */ +#if !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR) +#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) +#endif + int cmp_tree_item(const void *_a, const void *_b) { const struct tree_item *a = _a, *b = _b; @@ -202,7 +193,7 @@ static PyObject *py_sorted_tree_items(PyObject *self, PyObject *args) } while (PyDict_Next(entries, &pos, &key, &value)) { - if (!PyString_Check(key)) { + if (!PyBytes_Check(key)) { PyErr_SetString(PyExc_TypeError, "Name is not a string"); goto error; } @@ -213,18 +204,18 @@ static PyObject *py_sorted_tree_items(PyObject *self, PyObject *args) } py_mode = PyTuple_GET_ITEM(value, 0); - if (!PyInt_Check(py_mode) && !PyLong_Check(py_mode)) { + if (!PyLong_Check(py_mode)) { PyErr_SetString(PyExc_TypeError, "Mode is not an integral type"); goto error; } py_sha = PyTuple_GET_ITEM(value, 1); - if (!PyString_Check(py_sha)) { + if (!PyBytes_Check(py_sha)) { PyErr_SetString(PyExc_TypeError, "SHA is not a string"); goto error; } - qsort_entries[n].name = PyString_AS_STRING(key); - qsort_entries[n].mode = PyInt_AsLong(py_mode); + qsort_entries[n].name = PyBytes_AS_STRING(key); + qsort_entries[n].mode = PyLong_AsLong(py_mode); qsort_entries[n].tuple = PyObject_CallFunctionObjArgs( tree_entry_cls, key, py_mode, py_sha, NULL); @@ -267,7 +258,6 @@ moduleinit(void) { PyObject *m, *objects_mod, *errors_mod; -#if PY_MAJOR_VERSION >= 3 static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "_objects", /* m_name */ @@ -280,9 +270,6 @@ moduleinit(void) NULL, /* m_free */ }; m = PyModule_Create(&moduledef); -#else - m = Py_InitModule3("_objects", py_objects_methods, NULL); -#endif if (m == NULL) { return NULL; } @@ -315,16 +302,8 @@ moduleinit(void) return m; } -#if PY_MAJOR_VERSION >= 3 PyMODINIT_FUNC PyInit__objects(void) { return moduleinit(); } -#else -PyMODINIT_FUNC -init_objects(void) -{ - moduleinit(); -} -#endif diff --git a/ext_libs/dulwich/_pack.c b/ext_libs/dulwich/_pack.c index f2ef0649..3ee9828d 100644 --- a/ext_libs/dulwich/_pack.c +++ b/ext_libs/dulwich/_pack.c @@ -22,27 +22,14 @@ #include #include -#if PY_MAJOR_VERSION >= 3 -#define PyInt_FromLong PyLong_FromLong -#define PyString_AS_STRING PyBytes_AS_STRING -#define PyString_AS_STRING PyBytes_AS_STRING -#define PyString_Check PyBytes_Check -#define PyString_CheckExact PyBytes_CheckExact -#define PyString_FromStringAndSize PyBytes_FromStringAndSize -#define PyString_FromString PyBytes_FromString -#define PyString_GET_SIZE PyBytes_GET_SIZE -#define PyString_Size PyBytes_Size -#define _PyString_Join _PyBytes_Join -#endif - static PyObject *PyExc_ApplyDeltaError = NULL; static int py_is_sha(PyObject *sha) { - if (!PyString_CheckExact(sha)) + if (!PyBytes_CheckExact(sha)) return 0; - if (PyString_Size(sha) != 20) + if (PyBytes_Size(sha) != 20) return 0; return 1; @@ -67,18 +54,18 @@ static size_t get_delta_header_size(uint8_t *delta, size_t *index, size_t length static PyObject *py_chunked_as_string(PyObject *py_buf) { if (PyList_Check(py_buf)) { - PyObject *sep = PyString_FromString(""); + PyObject *sep = PyBytes_FromString(""); if (sep == NULL) { PyErr_NoMemory(); return NULL; } - py_buf = _PyString_Join(sep, py_buf); + py_buf = _PyBytes_Join(sep, py_buf); Py_DECREF(sep); if (py_buf == NULL) { PyErr_NoMemory(); return NULL; } - } else if (PyString_Check(py_buf)) { + } else if (PyBytes_Check(py_buf)) { Py_INCREF(py_buf); } else { PyErr_SetString(PyExc_TypeError, @@ -111,11 +98,11 @@ static PyObject *py_apply_delta(PyObject *self, PyObject *args) return NULL; } - src_buf = (uint8_t *)PyString_AS_STRING(py_src_buf); - src_buf_len = (size_t)PyString_GET_SIZE(py_src_buf); + src_buf = (uint8_t *)PyBytes_AS_STRING(py_src_buf); + src_buf_len = (size_t)PyBytes_GET_SIZE(py_src_buf); - delta = (uint8_t *)PyString_AS_STRING(py_delta); - delta_len = (size_t)PyString_GET_SIZE(py_delta); + delta = (uint8_t *)PyBytes_AS_STRING(py_delta); + delta_len = (size_t)PyBytes_GET_SIZE(py_delta); index = 0; src_size = get_delta_header_size(delta, &index, delta_len); @@ -127,14 +114,14 @@ static PyObject *py_apply_delta(PyObject *self, PyObject *args) return NULL; } dest_size = get_delta_header_size(delta, &index, delta_len); - ret = PyString_FromStringAndSize(NULL, dest_size); + ret = PyBytes_FromStringAndSize(NULL, dest_size); if (ret == NULL) { PyErr_NoMemory(); Py_DECREF(py_src_buf); Py_DECREF(py_delta); return NULL; } - out = (uint8_t *)PyString_AS_STRING(ret); + out = (uint8_t *)PyBytes_AS_STRING(ret); while (index < delta_len) { uint8_t cmd = delta[index]; index++; @@ -208,13 +195,8 @@ static PyObject *py_bisect_find_sha(PyObject *self, PyObject *args) char *sha; Py_ssize_t sha_len; int start, end; -#if PY_MAJOR_VERSION >= 3 if (!PyArg_ParseTuple(args, "iiy#O", &start, &end, &sha, &sha_len, &unpack_name)) -#else - if (!PyArg_ParseTuple(args, "iis#O", &start, &end, - &sha, &sha_len, &unpack_name)) -#endif return NULL; if (sha_len != 20) { @@ -239,14 +221,14 @@ static PyObject *py_bisect_find_sha(PyObject *self, PyObject *args) Py_DECREF(file_sha); return NULL; } - cmp = memcmp(PyString_AS_STRING(file_sha), sha, 20); + cmp = memcmp(PyBytes_AS_STRING(file_sha), sha, 20); Py_DECREF(file_sha); if (cmp < 0) start = i + 1; else if (cmp > 0) end = i - 1; else { - return PyInt_FromLong(i); + return PyLong_FromLong(i); } } Py_RETURN_NONE; @@ -265,7 +247,6 @@ moduleinit(void) PyObject *m; PyObject *errors_module; -#if PY_MAJOR_VERSION >= 3 static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "_pack", /* m_name */ @@ -277,7 +258,6 @@ moduleinit(void) NULL, /* m_clear*/ NULL, /* m_free */ }; -#endif errors_module = PyImport_ImportModule("dulwich.errors"); if (errors_module == NULL) @@ -288,27 +268,15 @@ moduleinit(void) if (PyExc_ApplyDeltaError == NULL) return NULL; -#if PY_MAJOR_VERSION >= 3 m = PyModule_Create(&moduledef); -#else - m = Py_InitModule3("_pack", py_pack_methods, NULL); -#endif if (m == NULL) return NULL; return m; } -#if PY_MAJOR_VERSION >= 3 PyMODINIT_FUNC PyInit__pack(void) { return moduleinit(); } -#else -PyMODINIT_FUNC -init_pack(void) -{ - moduleinit(); -} -#endif diff --git a/ext_libs/dulwich/cli.py b/ext_libs/dulwich/cli.py new file mode 100644 index 00000000..0ef80d08 --- /dev/null +++ b/ext_libs/dulwich/cli.py @@ -0,0 +1,732 @@ +#!/usr/bin/python3 -u +# +# dulwich - Simple command-line interface to Dulwich +# Copyright (C) 2008-2011 Jelmer Vernooij +# vim: expandtab +# +# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU +# General Public License as public by the Free Software Foundation; version 2.0 +# or (at your option) any later version. You can redistribute it and/or +# modify it under the terms of either of these two licenses. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# You should have received a copy of the licenses; if not, see +# for a copy of the GNU General Public License +# and for a copy of the Apache +# License, Version 2.0. +# + +"""Simple command-line interface to Dulwich> + +This is a very simple command-line wrapper for Dulwich. It is by +no means intended to be a full-blown Git command-line interface but just +a way to test Dulwich. +""" + +import os +import sys +from getopt import getopt +import optparse +import signal +from typing import Dict, Type + +from dulwich import porcelain +from dulwich.client import get_transport_and_path +from dulwich.errors import ApplyDeltaError +from dulwich.index import Index +from dulwich.pack import Pack, sha_to_hex +from dulwich.patch import write_tree_diff +from dulwich.repo import Repo + + +def signal_int(signal, frame): + sys.exit(1) + + +def signal_quit(signal, frame): + import pdb + pdb.set_trace() + + +class Command(object): + """A Dulwich subcommand.""" + + def run(self, args): + """Run the command.""" + raise NotImplementedError(self.run) + + +class cmd_archive(Command): + + def run(self, args): + parser = optparse.OptionParser() + parser.add_option("--remote", type=str, + help="Retrieve archive from specified remote repo") + options, args = parser.parse_args(args) + committish = args.pop(0) + if options.remote: + client, path = get_transport_and_path(options.remote) + client.archive( + path, committish, sys.stdout.write, + write_error=sys.stderr.write) + else: + porcelain.archive( + '.', committish, outstream=sys.stdout, + errstream=sys.stderr) + + +class cmd_add(Command): + + def run(self, args): + opts, args = getopt(args, "", []) + + porcelain.add(".", paths=args) + + +class cmd_rm(Command): + + def run(self, args): + opts, args = getopt(args, "", []) + + porcelain.rm(".", paths=args) + + +class cmd_fetch_pack(Command): + + def run(self, args): + opts, args = getopt(args, "", ["all"]) + opts = dict(opts) + client, path = get_transport_and_path(args.pop(0)) + r = Repo(".") + if "--all" in opts: + determine_wants = r.object_store.determine_wants_all + else: + def determine_wants(x): + return [y for y in args if y not in r.object_store] + client.fetch(path, r, determine_wants) + + +class cmd_fetch(Command): + + def run(self, args): + opts, args = getopt(args, "", []) + opts = dict(opts) + client, path = get_transport_and_path(args.pop(0)) + r = Repo(".") + refs = client.fetch(path, r, progress=sys.stdout.write) + print("Remote refs:") + for item in refs.items(): + print("%s -> %s" % item) + + +class cmd_fsck(Command): + + def run(self, args): + opts, args = getopt(args, "", []) + opts = dict(opts) + for (obj, msg) in porcelain.fsck('.'): + print("%s: %s" % (obj, msg)) + + +class cmd_log(Command): + + def run(self, args): + parser = optparse.OptionParser() + parser.add_option("--reverse", dest="reverse", action="store_true", + help="Reverse order in which entries are printed") + parser.add_option("--name-status", dest="name_status", + action="store_true", + help="Print name/status for each changed file") + options, args = parser.parse_args(args) + + porcelain.log(".", paths=args, reverse=options.reverse, + name_status=options.name_status, + outstream=sys.stdout) + + +class cmd_diff(Command): + + def run(self, args): + opts, args = getopt(args, "", []) + + if args == []: + print("Usage: dulwich diff COMMITID") + sys.exit(1) + + r = Repo(".") + commit_id = args[0] + commit = r[commit_id] + parent_commit = r[commit.parents[0]] + write_tree_diff( + sys.stdout, r.object_store, parent_commit.tree, commit.tree) + + +class cmd_dump_pack(Command): + + def run(self, args): + opts, args = getopt(args, "", []) + + if args == []: + print("Usage: dulwich dump-pack FILENAME") + sys.exit(1) + + basename, _ = os.path.splitext(args[0]) + x = Pack(basename) + print("Object names checksum: %s" % x.name()) + print("Checksum: %s" % sha_to_hex(x.get_stored_checksum())) + if not x.check(): + print("CHECKSUM DOES NOT MATCH") + print("Length: %d" % len(x)) + for name in x: + try: + print("\t%s" % x[name]) + except KeyError as k: + print("\t%s: Unable to resolve base %s" % (name, k)) + except ApplyDeltaError as e: + print("\t%s: Unable to apply delta: %r" % (name, e)) + + +class cmd_dump_index(Command): + + def run(self, args): + opts, args = getopt(args, "", []) + + if args == []: + print("Usage: dulwich dump-index FILENAME") + sys.exit(1) + + filename = args[0] + idx = Index(filename) + + for o in idx: + print(o, idx[o]) + + +class cmd_init(Command): + + def run(self, args): + opts, args = getopt(args, "", ["bare"]) + opts = dict(opts) + + if args == []: + path = os.getcwd() + else: + path = args[0] + + porcelain.init(path, bare=("--bare" in opts)) + + +class cmd_clone(Command): + + def run(self, args): + parser = optparse.OptionParser() + parser.add_option("--bare", dest="bare", + help="Whether to create a bare repository.", + action="store_true") + parser.add_option("--depth", dest="depth", + type=int, help="Depth at which to fetch") + options, args = parser.parse_args(args) + + if args == []: + print("usage: dulwich clone host:path [PATH]") + sys.exit(1) + + source = args.pop(0) + if len(args) > 0: + target = args.pop(0) + else: + target = None + + porcelain.clone(source, target, bare=options.bare, depth=options.depth) + + +class cmd_commit(Command): + + def run(self, args): + opts, args = getopt(args, "", ["message"]) + opts = dict(opts) + porcelain.commit(".", message=opts["--message"]) + + +class cmd_commit_tree(Command): + + def run(self, args): + opts, args = getopt(args, "", ["message"]) + if args == []: + print("usage: dulwich commit-tree tree") + sys.exit(1) + opts = dict(opts) + porcelain.commit_tree(".", tree=args[0], message=opts["--message"]) + + +class cmd_update_server_info(Command): + + def run(self, args): + porcelain.update_server_info(".") + + +class cmd_symbolic_ref(Command): + + def run(self, args): + opts, args = getopt(args, "", ["ref-name", "force"]) + if not args: + print("Usage: dulwich symbolic-ref REF_NAME [--force]") + sys.exit(1) + + ref_name = args.pop(0) + porcelain.symbolic_ref(".", ref_name=ref_name, force='--force' in args) + + +class cmd_show(Command): + + def run(self, args): + opts, args = getopt(args, "", []) + porcelain.show(".", args) + + +class cmd_diff_tree(Command): + + def run(self, args): + opts, args = getopt(args, "", []) + if len(args) < 2: + print("Usage: dulwich diff-tree OLD-TREE NEW-TREE") + sys.exit(1) + porcelain.diff_tree(".", args[0], args[1]) + + +class cmd_rev_list(Command): + + def run(self, args): + opts, args = getopt(args, "", []) + if len(args) < 1: + print('Usage: dulwich rev-list COMMITID...') + sys.exit(1) + porcelain.rev_list('.', args) + + +class cmd_tag(Command): + + def run(self, args): + parser = optparse.OptionParser() + parser.add_option( + "-a", "--annotated", help="Create an annotated tag.", + action="store_true") + parser.add_option( + "-s", "--sign", help="Sign the annotated tag.", + action="store_true") + options, args = parser.parse_args(args) + porcelain.tag_create( + '.', args[0], annotated=options.annotated, + sign=options.sign) + + +class cmd_repack(Command): + + def run(self, args): + opts, args = getopt(args, "", []) + opts = dict(opts) + porcelain.repack('.') + + +class cmd_reset(Command): + + def run(self, args): + opts, args = getopt(args, "", ["hard", "soft", "mixed"]) + opts = dict(opts) + mode = "" + if "--hard" in opts: + mode = "hard" + elif "--soft" in opts: + mode = "soft" + elif "--mixed" in opts: + mode = "mixed" + porcelain.reset('.', mode=mode, *args) + + +class cmd_daemon(Command): + + def run(self, args): + from dulwich import log_utils + from dulwich.protocol import TCP_GIT_PORT + parser = optparse.OptionParser() + parser.add_option("-l", "--listen_address", dest="listen_address", + default="localhost", + help="Binding IP address.") + parser.add_option("-p", "--port", dest="port", type=int, + default=TCP_GIT_PORT, + help="Binding TCP port.") + options, args = parser.parse_args(args) + + log_utils.default_logging_config() + if len(args) >= 1: + gitdir = args[0] + else: + gitdir = '.' + from dulwich import porcelain + porcelain.daemon(gitdir, address=options.listen_address, + port=options.port) + + +class cmd_web_daemon(Command): + + def run(self, args): + from dulwich import log_utils + parser = optparse.OptionParser() + parser.add_option("-l", "--listen_address", dest="listen_address", + default="", + help="Binding IP address.") + parser.add_option("-p", "--port", dest="port", type=int, + default=8000, + help="Binding TCP port.") + options, args = parser.parse_args(args) + + log_utils.default_logging_config() + if len(args) >= 1: + gitdir = args[0] + else: + gitdir = '.' + from dulwich import porcelain + porcelain.web_daemon(gitdir, address=options.listen_address, + port=options.port) + + +class cmd_write_tree(Command): + + def run(self, args): + parser = optparse.OptionParser() + options, args = parser.parse_args(args) + sys.stdout.write('%s\n' % porcelain.write_tree('.')) + + +class cmd_receive_pack(Command): + + def run(self, args): + parser = optparse.OptionParser() + options, args = parser.parse_args(args) + if len(args) >= 1: + gitdir = args[0] + else: + gitdir = '.' + porcelain.receive_pack(gitdir) + + +class cmd_upload_pack(Command): + + def run(self, args): + parser = optparse.OptionParser() + options, args = parser.parse_args(args) + if len(args) >= 1: + gitdir = args[0] + else: + gitdir = '.' + porcelain.upload_pack(gitdir) + + +class cmd_status(Command): + + def run(self, args): + parser = optparse.OptionParser() + options, args = parser.parse_args(args) + if len(args) >= 1: + gitdir = args[0] + else: + gitdir = '.' + status = porcelain.status(gitdir) + if any(names for (kind, names) in status.staged.items()): + sys.stdout.write("Changes to be committed:\n\n") + for kind, names in status.staged.items(): + for name in names: + sys.stdout.write("\t%s: %s\n" % ( + kind, name.decode(sys.getfilesystemencoding()))) + sys.stdout.write("\n") + if status.unstaged: + sys.stdout.write("Changes not staged for commit:\n\n") + for name in status.unstaged: + sys.stdout.write( + "\t%s\n" % name.decode(sys.getfilesystemencoding())) + sys.stdout.write("\n") + if status.untracked: + sys.stdout.write("Untracked files:\n\n") + for name in status.untracked: + sys.stdout.write("\t%s\n" % name) + sys.stdout.write("\n") + + +class cmd_ls_remote(Command): + + def run(self, args): + opts, args = getopt(args, '', []) + if len(args) < 1: + print('Usage: dulwich ls-remote URL') + sys.exit(1) + refs = porcelain.ls_remote(args[0]) + for ref in sorted(refs): + sys.stdout.write("%s\t%s\n" % (ref, refs[ref])) + + +class cmd_ls_tree(Command): + + def run(self, args): + parser = optparse.OptionParser() + parser.add_option("-r", "--recursive", action="store_true", + help="Recusively list tree contents.") + parser.add_option("--name-only", action="store_true", + help="Only display name.") + options, args = parser.parse_args(args) + try: + treeish = args.pop(0) + except IndexError: + treeish = None + porcelain.ls_tree( + '.', treeish, outstream=sys.stdout, recursive=options.recursive, + name_only=options.name_only) + + +class cmd_pack_objects(Command): + + def run(self, args): + opts, args = getopt(args, '', ['stdout']) + opts = dict(opts) + if len(args) < 1 and '--stdout' not in args: + print('Usage: dulwich pack-objects basename') + sys.exit(1) + object_ids = [line.strip() for line in sys.stdin.readlines()] + basename = args[0] + if '--stdout' in opts: + packf = getattr(sys.stdout, 'buffer', sys.stdout) + idxf = None + close = [] + else: + packf = open(basename + '.pack', 'w') + idxf = open(basename + '.idx', 'w') + close = [packf, idxf] + porcelain.pack_objects('.', object_ids, packf, idxf) + for f in close: + f.close() + + +class cmd_pull(Command): + + def run(self, args): + parser = optparse.OptionParser() + options, args = parser.parse_args(args) + try: + from_location = args[0] + except IndexError: + from_location = None + porcelain.pull('.', from_location) + + +class cmd_push(Command): + + def run(self, args): + parser = optparse.OptionParser() + options, args = parser.parse_args(args) + if len(args) < 2: + print("Usage: dulwich push TO-LOCATION REFSPEC..") + sys.exit(1) + to_location = args[0] + refspecs = args[1:] + porcelain.push('.', to_location, refspecs) + + +class cmd_remote_add(Command): + + def run(self, args): + parser = optparse.OptionParser() + options, args = parser.parse_args(args) + porcelain.remote_add('.', args[0], args[1]) + + +class SuperCommand(Command): + + subcommands = {} # type: Dict[str, Type[Command]] + + def run(self, args): + if not args: + print("Supported subcommands: %s" % + ', '.join(self.subcommands.keys())) + return False + cmd = args[0] + try: + cmd_kls = self.subcommands[cmd] + except KeyError: + print('No such subcommand: %s' % args[0]) + return False + return cmd_kls().run(args[1:]) + + +class cmd_remote(SuperCommand): + + subcommands = { + "add": cmd_remote_add, + } + + +class cmd_check_ignore(Command): + + def run(self, args): + parser = optparse.OptionParser() + options, args = parser.parse_args(args) + ret = 1 + for path in porcelain.check_ignore('.', args): + print(path) + ret = 0 + return ret + + +class cmd_check_mailmap(Command): + + def run(self, args): + parser = optparse.OptionParser() + options, args = parser.parse_args(args) + for arg in args: + canonical_identity = porcelain.check_mailmap('.', arg) + print(canonical_identity) + + +class cmd_stash_list(Command): + + def run(self, args): + parser = optparse.OptionParser() + options, args = parser.parse_args(args) + for i, entry in porcelain.stash_list('.'): + print("stash@{%d}: %s" % (i, entry.message.rstrip('\n'))) + + +class cmd_stash_push(Command): + + def run(self, args): + parser = optparse.OptionParser() + options, args = parser.parse_args(args) + porcelain.stash_push('.') + print("Saved working directory and index state") + + +class cmd_stash_pop(Command): + + def run(self, args): + parser = optparse.OptionParser() + options, args = parser.parse_args(args) + porcelain.stash_pop('.') + print("Restrored working directory and index state") + + +class cmd_stash(SuperCommand): + + subcommands = { + "list": cmd_stash_list, + "pop": cmd_stash_pop, + "push": cmd_stash_push, + } + + +class cmd_ls_files(Command): + + def run(self, args): + parser = optparse.OptionParser() + options, args = parser.parse_args(args) + for name in porcelain.ls_files('.'): + print(name) + + +class cmd_describe(Command): + + def run(self, args): + parser = optparse.OptionParser() + options, args = parser.parse_args(args) + print(porcelain.describe('.')) + + +class cmd_help(Command): + + def run(self, args): + parser = optparse.OptionParser() + parser.add_option("-a", "--all", dest="all", + action="store_true", + help="List all commands.") + options, args = parser.parse_args(args) + + if options.all: + print('Available commands:') + for cmd in sorted(commands): + print(' %s' % cmd) + else: + print("""\ +The dulwich command line tool is currently a very basic frontend for the +Dulwich python module. For full functionality, please see the API reference. + +For a list of supported commands, see 'dulwich help -a'. +""") + + +commands = { + "add": cmd_add, + "archive": cmd_archive, + "check-ignore": cmd_check_ignore, + "check-mailmap": cmd_check_mailmap, + "clone": cmd_clone, + "commit": cmd_commit, + "commit-tree": cmd_commit_tree, + "describe": cmd_describe, + "daemon": cmd_daemon, + "diff": cmd_diff, + "diff-tree": cmd_diff_tree, + "dump-pack": cmd_dump_pack, + "dump-index": cmd_dump_index, + "fetch-pack": cmd_fetch_pack, + "fetch": cmd_fetch, + "fsck": cmd_fsck, + "help": cmd_help, + "init": cmd_init, + "log": cmd_log, + "ls-files": cmd_ls_files, + "ls-remote": cmd_ls_remote, + "ls-tree": cmd_ls_tree, + "pack-objects": cmd_pack_objects, + "pull": cmd_pull, + "push": cmd_push, + "receive-pack": cmd_receive_pack, + "remote": cmd_remote, + "repack": cmd_repack, + "reset": cmd_reset, + "rev-list": cmd_rev_list, + "rm": cmd_rm, + "show": cmd_show, + "stash": cmd_stash, + "status": cmd_status, + "symbolic-ref": cmd_symbolic_ref, + "tag": cmd_tag, + "update-server-info": cmd_update_server_info, + "upload-pack": cmd_upload_pack, + "web-daemon": cmd_web_daemon, + "write-tree": cmd_write_tree, + } + + +def main(argv=None): + if len(argv) < 1: + print("Usage: dulwich <%s> [OPTIONS...]" % ("|".join(commands.keys()))) + return 1 + + cmd = argv[0] + try: + cmd_kls = commands[cmd] + except KeyError: + print("No such subcommand: %s" % cmd) + return 1 + # TODO(jelmer): Return non-0 on errors + return cmd_kls().run(argv[1:]) + + +if __name__ == '__main__': + if 'DULWICH_PDB' in os.environ and getattr(signal, 'SIGQUIT', None): + signal.signal(signal.SIGQUIT, signal_quit) # type: ignore + signal.signal(signal.SIGINT, signal_int) + + sys.exit(main(sys.argv[1:])) diff --git a/ext_libs/dulwich/client.py b/ext_libs/dulwich/client.py index 5dd497e5..9453b13a 100644 --- a/ext_libs/dulwich/client.py +++ b/ext_libs/dulwich/client.py @@ -30,39 +30,38 @@ * quiet * report-status * delete-refs + * shallow Known capabilities that are not supported: - * shallow * no-progress * include-tag """ from contextlib import closing from io import BytesIO, BufferedReader +import os import select import socket import subprocess import sys - -try: - from urllib import quote as urlquote - from urllib import unquote as urlunquote -except ImportError: - from urllib.parse import quote as urlquote - from urllib.parse import unquote as urlunquote - -try: - import urlparse -except ImportError: - import urllib.parse as urlparse +from typing import Optional, Dict, Callable, Set + +from urllib.parse import ( + quote as urlquote, + unquote as urlunquote, + urlparse, + urljoin, + urlunsplit, + urlunparse, + ) import dulwich +from dulwich.config import get_xdg_config_home_path from dulwich.errors import ( GitProtocolError, NotGitRepository, SendPackError, - UpdateRefsError, ) from dulwich.protocol import ( HangupException, @@ -72,6 +71,7 @@ extract_capability_names, CAPABILITY_AGENT, CAPABILITY_DELETE_REFS, + CAPABILITY_INCLUDE_TAG, CAPABILITY_MULTI_ACK, CAPABILITY_MULTI_ACK_DETAILED, CAPABILITY_OFS_DELTA, @@ -120,6 +120,14 @@ def __init__(self, wants): "requested wants not in server provided refs: %r" % wants) +class HTTPUnauthorized(Exception): + """Raised when authentication fails.""" + + def __init__(self, www_authenticate): + Exception.__init__(self, "No valid credentials provided") + self.www_authenticate = www_authenticate + + def _fileno_can_read(fileno): """Check if a file descriptor is readable. """ @@ -144,7 +152,9 @@ def _win32_peek_avail(handle): UPLOAD_CAPABILITIES = ([CAPABILITY_THIN_PACK, CAPABILITY_MULTI_ACK, CAPABILITY_MULTI_ACK_DETAILED, CAPABILITY_SHALLOW] + COMMON_CAPABILITIES) -RECEIVE_CAPABILITIES = [CAPABILITY_REPORT_STATUS] + COMMON_CAPABILITIES +RECEIVE_CAPABILITIES = ( + [CAPABILITY_REPORT_STATUS, CAPABILITY_DELETE_REFS] + + COMMON_CAPABILITIES) class ReportStatusParser(object): @@ -153,7 +163,6 @@ class ReportStatusParser(object): def __init__(self): self._done = False self._pack_status = None - self._ref_status_ok = True self._ref_statuses = [] def check(self): @@ -161,30 +170,24 @@ def check(self): Raises: SendPackError: Raised when the server could not unpack - UpdateRefsError: Raised when refs could not be updated + Returns: + iterator over refs """ if self._pack_status not in (b'unpack ok', None): raise SendPackError(self._pack_status) - if not self._ref_status_ok: - ref_status = {} - ok = set() - for status in self._ref_statuses: - if b' ' not in status: - # malformed response, move on to the next one - continue - status, ref = status.split(b' ', 1) - - if status == b'ng': - if b' ' in ref: - ref, status = ref.split(b' ', 1) - else: - ok.add(ref) - ref_status[ref] = status - # TODO(jelmer): don't assume encoding of refs is ascii. - raise UpdateRefsError(', '.join([ - refname.decode('ascii') for refname in ref_status - if refname not in ok]) + - ' failed to update', ref_status=ref_status) + for status in self._ref_statuses: + try: + status, rest = status.split(b' ', 1) + except ValueError: + # malformed response, move on to the next one + continue + if status == b'ng': + ref, error = rest.split(b' ', 1) + yield ref, error.decode('utf-8') + elif status == b'ok': + yield rest, None + else: + raise GitProtocolError('invalid ref status %r' % status) def handle_packet(self, pkt): """Handle a packet. @@ -203,8 +206,6 @@ def handle_packet(self, pkt): else: ref_status = pkt.strip() self._ref_statuses.append(ref_status) - if not ref_status.startswith(b'ok '): - self._ref_status_ok = False def read_pkt_refs(proto): @@ -236,8 +237,8 @@ class FetchPackResult(object): """ _FORWARDED_ATTRS = [ - 'clear', 'copy', 'fromkeys', 'get', 'has_key', 'items', - 'iteritems', 'iterkeys', 'itervalues', 'keys', 'pop', 'popitem', + 'clear', 'copy', 'fromkeys', 'get', 'items', + 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values', 'viewitems', 'viewkeys', 'viewvalues'] @@ -290,6 +291,66 @@ def __repr__(self): self.__class__.__name__, self.refs, self.symrefs, self.agent) +class SendPackResult(object): + """Result of a upload-pack operation. + + Attributes: + refs: Dictionary with all remote refs + agent: User agent string + ref_status: Optional dictionary mapping ref name to error message (if it + failed to update), or None if it was updated successfully + """ + + _FORWARDED_ATTRS = [ + 'clear', 'copy', 'fromkeys', 'get', 'items', + 'keys', 'pop', 'popitem', + 'setdefault', 'update', 'values', 'viewitems', 'viewkeys', + 'viewvalues'] + + def __init__(self, refs, agent=None, ref_status=None): + self.refs = refs + self.agent = agent + self.ref_status = ref_status + + def _warn_deprecated(self): + import warnings + warnings.warn( + "Use SendPackResult.refs instead.", + DeprecationWarning, stacklevel=3) + + def __eq__(self, other): + if isinstance(other, dict): + self._warn_deprecated() + return self.refs == other + return self.refs == other.refs and self.agent == other.agent + + def __contains__(self, name): + self._warn_deprecated() + return name in self.refs + + def __getitem__(self, name): + self._warn_deprecated() + return self.refs[name] + + def __len__(self): + self._warn_deprecated() + return len(self.refs) + + def __iter__(self): + self._warn_deprecated() + return iter(self.refs) + + def __getattribute__(self, name): + if name in type(self)._FORWARDED_ATTRS: + self._warn_deprecated() + return getattr(self.refs, name) + return super(SendPackResult, self).__getattribute__(name) + + def __repr__(self): + return "%s(%r, %r)" % ( + self.__class__.__name__, self.refs, self.agent) + + def _read_shallow_updates(proto): new_shallow = set() new_unshallow = set() @@ -310,13 +371,16 @@ def _read_shallow_updates(proto): class GitClient(object): """Git smart server client.""" - def __init__(self, thin_packs=True, report_activity=None, quiet=False): + def __init__(self, thin_packs=True, report_activity=None, quiet=False, + include_tags=False): """Create a new GitClient instance. Args: thin_packs: Whether or not thin packs should be retrieved report_activity: Optional callback for reporting transport activity. + include_tags: send annotated tags when sending the objects they point + to """ self._report_activity = report_activity self._report_status_parser = None @@ -328,6 +392,8 @@ def __init__(self, thin_packs=True, report_activity=None, quiet=False): self._send_capabilities.add(CAPABILITY_QUIET) if not thin_packs: self._fetch_capabilities.remove(CAPABILITY_THIN_PACK) + if include_tags: + self._fetch_capabilities.add(CAPABILITY_INCLUDE_TAG) def get_url(self, path): """Retrieves full url to given path. @@ -346,7 +412,7 @@ def from_parsedurl(cls, parsedurl, **kwargs): """Create an instance of this client from a urlparse.parsed object. Args: - parsedurl: Result of urlparse.urlparse() + parsedurl: Result of urlparse() Returns: A `GitClient` object @@ -367,13 +433,10 @@ def send_pack(self, path, update_refs, generate_pack_data, progress: Optional progress function Returns: - new_refs dictionary containing the changes that were made - {refname: new_ref}, including deleted refs. + SendPackResult object Raises: SendPackError: if server rejects the pack data - UpdateRefsError: if the server supports report-status - and rejects ref updates """ raise NotImplementedError(self.send_pack) @@ -454,43 +517,6 @@ def get_refs(self, path): """ raise NotImplementedError(self.get_refs) - def _parse_status_report(self, proto): - unpack = proto.read_pkt_line().strip() - if unpack != b'unpack ok': - st = True - # flush remaining error data - while st is not None: - st = proto.read_pkt_line() - raise SendPackError(unpack) - statuses = [] - errs = False - ref_status = proto.read_pkt_line() - while ref_status: - ref_status = ref_status.strip() - statuses.append(ref_status) - if not ref_status.startswith(b'ok '): - errs = True - ref_status = proto.read_pkt_line() - - if errs: - ref_status = {} - ok = set() - for status in statuses: - if b' ' not in status: - # malformed response, move on to the next one - continue - status, ref = status.split(b' ', 1) - - if status == b'ng': - if b' ' in ref: - ref, status = ref.split(b' ', 1) - else: - ok.add(ref) - ref_status[ref] = status - raise UpdateRefsError(', '.join([ - refname for refname in ref_status if refname not in ok]) + - b' failed to update', ref_status=ref_status) - def _read_side_band64k_data(self, proto, channel_callbacks): """Read per-channel data. @@ -512,6 +538,11 @@ def _read_side_band64k_data(self, proto, channel_callbacks): if cb is not None: cb(pkt) + @staticmethod + def _should_send_pack(new_refs): + # The packfile MUST NOT be sent if the only command used is delete. + return any(sha != ZERO_SHA for sha in new_refs.values()) + def _handle_receive_pack_head(self, proto, capabilities, old_refs, new_refs): """Handle the head of a 'git-receive-pack' request. @@ -523,8 +554,7 @@ def _handle_receive_pack_head(self, proto, capabilities, old_refs, new_refs: Refs to change Returns: - have, want) tuple - + (have, want) tuple """ want = [] have = [x for x in old_refs.values() if not x == ZERO_SHA] @@ -549,7 +579,7 @@ def _handle_receive_pack_head(self, proto, capabilities, old_refs, else: proto.write_pkt_line( old_sha1 + b' ' + new_sha1 + b' ' + refname + b'\0' + - b' '.join(capabilities)) + b' '.join(sorted(capabilities))) sent_capabilities = True if new_sha1 not in have and new_sha1 != ZERO_SHA: want.append(new_sha1) @@ -559,13 +589,21 @@ def _handle_receive_pack_head(self, proto, capabilities, old_refs, def _negotiate_receive_pack_capabilities(self, server_capabilities): negotiated_capabilities = ( self._send_capabilities & server_capabilities) + agent = None + for capability in server_capabilities: + k, v = parse_capability(capability) + if k == CAPABILITY_AGENT: + agent = v unknown_capabilities = ( # noqa: F841 extract_capability_names(server_capabilities) - KNOWN_RECEIVE_CAPABILITIES) # TODO(jelmer): warn about unknown capabilities - return negotiated_capabilities + return negotiated_capabilities, agent - def _handle_receive_pack_tail(self, proto, capabilities, progress=None): + def _handle_receive_pack_tail( + self, proto: Protocol, capabilities: Set[bytes], + progress: Callable[[bytes], None] = None + ) -> Optional[Dict[bytes, Optional[str]]]: """Handle the tail of a 'git-receive-pack' request. Args: @@ -574,7 +612,9 @@ def _handle_receive_pack_tail(self, proto, capabilities, progress=None): progress: Optional progress reporting function Returns: - + dict mapping ref name to: + error message if the ref failed to update + None if it was updated successfully """ if CAPABILITY_SIDE_BAND_64K in capabilities: if progress is None: @@ -590,7 +630,9 @@ def progress(x): for pkt in proto.read_pkt_seq(): self._report_status_parser.handle_packet(pkt) if self._report_status_parser is not None: - self._report_status_parser.check() + return dict(self._report_status_parser.check()) + + return None def _negotiate_upload_pack_capabilities(self, server_capabilities): unknown_capabilities = ( # noqa: F841 @@ -629,7 +671,7 @@ def _handle_upload_pack_head(self, proto, capabilities, graph_walker, """ assert isinstance(wants, list) and isinstance(wants[0], bytes) proto.write_pkt_line(COMMAND_WANT + b' ' + wants[0] + b' ' + - b' '.join(capabilities) + b'\n') + b' '.join(sorted(capabilities)) + b'\n') for want in wants[1:]: proto.write_pkt_line(COMMAND_WANT + b' ' + want + b'\n') if depth not in (0, None) or getattr(graph_walker, 'shallow', None): @@ -639,8 +681,9 @@ def _handle_upload_pack_head(self, proto, capabilities, graph_walker, "depth") for sha in graph_walker.shallow: proto.write_pkt_line(COMMAND_SHALLOW + b' ' + sha + b'\n') - proto.write_pkt_line(COMMAND_DEEPEN + b' ' + - str(depth).encode('ascii') + b'\n') + if depth is not None: + proto.write_pkt_line(COMMAND_DEEPEN + b' ' + + str(depth).encode('ascii') + b'\n') proto.write_pkt_line(None) if can_read is not None: (new_shallow, new_unshallow) = _read_shallow_updates(proto) @@ -728,15 +771,15 @@ def check_wants(wants, refs): raise InvalidWants(missing) -def remote_error_from_stderr(stderr): +def _remote_error_from_stderr(stderr): if stderr is None: return HangupException() - for l in stderr.readlines(): - if l.startswith(b'ERROR: '): + lines = [line.rstrip(b'\n') for line in stderr.readlines()] + for line in lines: + if line.startswith(b'ERROR: '): return GitProtocolError( - l[len(b'ERROR: '):].decode('utf-8', 'replace')) - return GitProtocolError(l.decode('utf-8', 'replace')) - return HangupException() + line[len(b'ERROR: '):].decode('utf-8', 'replace')) + return HangupException(lines) class TraditionalGitClient(GitClient): @@ -777,13 +820,10 @@ def send_pack(self, path, update_refs, generate_pack_data, progress: Optional callback called with progress updates Returns: - new_refs dictionary containing the changes that were made - {refname: new_ref}, including deleted refs. + SendPackResult Raises: SendPackError: if server rejects the pack data - UpdateRefsError: if the server supports report-status - and rejects ref updates """ proto, unused_can_read, stderr = self._connect(b'receive-pack', path) @@ -791,8 +831,8 @@ def send_pack(self, path, update_refs, generate_pack_data, try: old_refs, server_capabilities = read_pkt_refs(proto) except HangupException: - raise remote_error_from_stderr(stderr) - negotiated_capabilities = \ + raise _remote_error_from_stderr(stderr) + negotiated_capabilities, agent = \ self._negotiate_receive_pack_capabilities(server_capabilities) if CAPABILITY_REPORT_STATUS in negotiated_capabilities: self._report_status_parser = ReportStatusParser() @@ -804,6 +844,10 @@ def send_pack(self, path, update_refs, generate_pack_data, proto.write_pkt_line(None) raise + if set(new_refs.items()).issubset(set(old_refs.items())): + proto.write_pkt_line(None) + return SendPackResult(new_refs, agent=agent, ref_status={}) + if CAPABILITY_DELETE_REFS not in server_capabilities: # Server does not support deletions. Fail later. new_refs = dict(orig_new_refs) @@ -811,41 +855,38 @@ def send_pack(self, path, update_refs, generate_pack_data, if sha == ZERO_SHA: if CAPABILITY_REPORT_STATUS in negotiated_capabilities: report_status_parser._ref_statuses.append( - b'ng ' + sha + + b'ng ' + ref + b' remote does not support deleting refs') report_status_parser._ref_status_ok = False del new_refs[ref] if new_refs is None: proto.write_pkt_line(None) - return old_refs + return SendPackResult(old_refs, agent=agent, ref_status={}) if len(new_refs) == 0 and len(orig_new_refs): # NOOP - Original new refs filtered out by policy proto.write_pkt_line(None) if report_status_parser is not None: - report_status_parser.check() - return old_refs + ref_status = dict(report_status_parser.check()) + else: + ref_status = None + return SendPackResult( + old_refs, agent=agent, ref_status=ref_status) (have, want) = self._handle_receive_pack_head( proto, negotiated_capabilities, old_refs, new_refs) - if (not want and - set(new_refs.items()).issubset(set(old_refs.items()))): - return new_refs + pack_data_count, pack_data = generate_pack_data( have, want, ofs_delta=(CAPABILITY_OFS_DELTA in negotiated_capabilities)) - dowrite = bool(pack_data_count) - dowrite = dowrite or any(old_refs.get(ref) != sha - for (ref, sha) in new_refs.items() - if sha != ZERO_SHA) - if dowrite: + if self._should_send_pack(new_refs): write_pack_data(proto.write_file(), pack_data_count, pack_data) - self._handle_receive_pack_tail( + ref_status = self._handle_receive_pack_tail( proto, negotiated_capabilities, progress) - return new_refs + return SendPackResult(new_refs, agent=agent, ref_status=ref_status) def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress=None, depth=None): @@ -870,7 +911,7 @@ def fetch_pack(self, path, determine_wants, graph_walker, pack_data, try: refs, server_capabilities = read_pkt_refs(proto) except HangupException: - raise remote_error_from_stderr(stderr) + raise _remote_error_from_stderr(stderr) negotiated_capabilities, symrefs, agent = ( self._negotiate_upload_pack_capabilities( server_capabilities)) @@ -907,7 +948,7 @@ def get_refs(self, path): try: refs, _ = read_pkt_refs(proto) except HangupException: - raise remote_error_from_stderr(stderr) + raise _remote_error_from_stderr(stderr) proto.write_pkt_line(None) return refs @@ -927,7 +968,7 @@ def archive(self, path, committish, write_data, progress=None, try: pkt = proto.read_pkt_line() except HangupException: - raise remote_error_from_stderr(stderr) + raise _remote_error_from_stderr(stderr) if pkt == b"NACK\n": return elif pkt == b"ACK\n": @@ -964,7 +1005,7 @@ def get_url(self, path): netloc = self._host if self._port is not None and self._port != TCP_GIT_PORT: netloc += ":%d" % self._port - return urlparse.urlunsplit(("git", netloc, path, '', '')) + return urlunsplit(("git", netloc, path, '', '')) def _connect(self, cmd, path): if not isinstance(cmd, bytes): @@ -1013,10 +1054,7 @@ class SubprocessWrapper(object): def __init__(self, proc): self.proc = proc - if sys.version_info[0] == 2: - self.read = proc.stdout.read - else: - self.read = BufferedReader(proc.stdout).read + self.read = BufferedReader(proc.stdout).read self.write = proc.stdin.write @property @@ -1094,7 +1132,7 @@ def __init__(self, thin_packs=True, report_activity=None, config=None): # Ignore the thin_packs argument def get_url(self, path): - return urlparse.urlunsplit(('file', '', path, '', '')) + return urlunsplit(('file', '', path, '', '')) @classmethod def from_parsedurl(cls, parsedurl, **kwargs): @@ -1104,7 +1142,7 @@ def from_parsedurl(cls, parsedurl, **kwargs): def _open_repo(cls, path): from dulwich.repo import Repo if not isinstance(path, str): - path = path.decode(sys.getfilesystemencoding()) + path = os.fsdecode(path) return closing(Repo(path)) def send_pack(self, path, update_refs, generate_pack_data, @@ -1116,18 +1154,14 @@ def send_pack(self, path, update_refs, generate_pack_data, update_refs: Function to determine changes to remote refs. Receive dict with existing remote refs, returns dict with changed refs (name -> sha, where sha=ZERO_SHA for deletions) - generate_pack_data: Function that can return a tuple with number of items and pack data to upload. progress: Optional progress function Returns: - new_refs dictionary containing the changes that were made - {refname: new_ref}, including deleted refs. + SendPackResult Raises: SendPackError: if server rejects the pack data - UpdateRefsError: if the server supports report-status - and rejects ref updates """ if not progress: @@ -1148,23 +1182,27 @@ def progress(x): if (not want and set(new_refs.items()).issubset(set(old_refs.items()))): - return new_refs + return SendPackResult(new_refs, ref_status={}) target.object_store.add_pack_data( *generate_pack_data(have, want, ofs_delta=True)) + ref_status = {} + for refname, new_sha1 in new_refs.items(): old_sha1 = old_refs.get(refname, ZERO_SHA) if new_sha1 != ZERO_SHA: if not target.refs.set_if_equals( refname, old_sha1, new_sha1): - progress('unable to set %s to %s' % - (refname, new_sha1)) + msg = 'unable to set %s to %s' % (refname, new_sha1) + progress(msg) + ref_status[refname] = msg else: if not target.refs.remove_if_equals(refname, old_sha1): progress('unable to remove %s' % refname) + ref_status[refname] = 'unable to remove' - return new_refs + return SendPackResult(new_refs, ref_status=ref_status) def fetch(self, path, target, determine_wants=None, progress=None, depth=None): @@ -1379,7 +1417,7 @@ def get_url(self, path): if self.username is not None: netloc = urlquote(self.username, '@/:') + "@" + netloc - return urlparse.urlunsplit(('ssh', netloc, path, '', '')) + return urlunsplit(('ssh', netloc, path, '', '')) @classmethod def from_parsedurl(cls, parsedurl, **kwargs): @@ -1419,7 +1457,8 @@ def default_user_agent_string(): return "git/dulwich/%s" % ".".join([str(x) for x in dulwich.__version__]) -def default_urllib3_manager(config, **override_kwargs): +def default_urllib3_manager(config, pool_manager_cls=None, + proxy_manager_cls=None, **override_kwargs): """Return `urllib3` connection pool manager. Honour detected proxy configurations. @@ -1429,8 +1468,9 @@ def default_urllib3_manager(config, **override_kwargs): kwargs: Additional arguments for urllib3.ProxyManager Returns: - urllib3.ProxyManager` instance for proxy configurations, - `urllib3.PoolManager` otherwise. + `pool_manager_cls` (defaults to `urllib3.ProxyManager`) instance for + proxy configurations, `proxy_manager_cls` (defaults to + `urllib3.PoolManager`) instance otherwise. """ proxy_server = user_agent = None @@ -1487,14 +1527,17 @@ def default_urllib3_manager(config, **override_kwargs): import urllib3 if proxy_server is not None: + if proxy_manager_cls is None: + proxy_manager_cls = urllib3.ProxyManager # `urllib3` requires a `str` object in both Python 2 and 3, while # `ConfigDict` coerces entries to `bytes` on Python 3. Compensate. if not isinstance(proxy_server, str): proxy_server = proxy_server.decode() - manager = urllib3.ProxyManager(proxy_server, headers=headers, - **kwargs) + manager = proxy_manager_cls(proxy_server, headers=headers, **kwargs) else: - manager = urllib3.PoolManager(headers=headers, **kwargs) + if pool_manager_cls is None: + pool_manager_cls = urllib3.PoolManager + manager = pool_manager_cls(headers=headers, **kwargs) return manager @@ -1540,7 +1583,7 @@ def from_parsedurl(cls, parsedurl, **kwargs): if parsedurl.username: netloc = "%s@%s" % (parsedurl.username, netloc) parsedurl = parsedurl._replace(netloc=netloc) - return cls(urlparse.urlunparse(parsedurl), **kwargs) + return cls(urlunparse(parsedurl), **kwargs) def __repr__(self): return "%s(%r, dumb=%r)" % ( @@ -1548,11 +1591,10 @@ def __repr__(self): def _get_url(self, path): if not isinstance(path, str): - # TODO(jelmer): this is unrelated to the local filesystem; - # This is not necessarily the right encoding to decode the path - # with. - path = path.decode(sys.getfilesystemencoding()) - return urlparse.urljoin(self._base_url, path).rstrip("/") + "/" + # urllib3.util.url._encode_invalid_chars() converts the path back + # to bytes using the utf-8 codec. + path = path.decode('utf-8') + return urljoin(self._base_url, path).rstrip("/") + "/" def _http_request(self, url, headers=None, data=None, allow_compression=False): @@ -1588,6 +1630,8 @@ def _http_request(self, url, headers=None, data=None, if resp.status == 404: raise NotGitRepository() + elif resp.status == 401: + raise HTTPUnauthorized(resp.getheader('WWW-Authenticate')) elif resp.status != 200: raise GitProtocolError("unexpected http resp %d for %s" % (resp.status, url)) @@ -1616,7 +1660,7 @@ def _discover_references(self, service, base_url): headers = {"Accept": "*/*"} if self.dumb is not True: tail += "?service=%s" % service.decode('ascii') - url = urlparse.urljoin(base_url, tail) + url = urljoin(base_url, tail) resp, read = self._http_request(url, headers, allow_compression=True) if resp.redirect_location: @@ -1648,7 +1692,7 @@ def _discover_references(self, service, base_url): def _smart_request(self, service, url, data): assert url[-1] == "/" - url = urlparse.urljoin(url, service) + url = urljoin(url, service) result_content_type = "application/x-%s-result" % service headers = { "Content-Type": "application/x-%s-request" % service, @@ -1668,27 +1712,24 @@ def send_pack(self, path, update_refs, generate_pack_data, Args: path: Repository path (as bytestring) update_refs: Function to determine changes to remote refs. - Receive dict with existing remote refs, returns dict with + Receives dict with existing remote refs, returns dict with changed refs (name -> sha, where sha=ZERO_SHA for deletions) generate_pack_data: Function that can return a tuple with number of elements and pack data to upload. progress: Optional progress function Returns: - new_refs dictionary containing the changes that were made - {refname: new_ref}, including deleted refs. + SendPackResult Raises: SendPackError: if server rejects the pack data - UpdateRefsError: if the server supports report-status - and rejects ref updates """ url = self._get_url(path) old_refs, server_capabilities, url = self._discover_references( b"git-receive-pack", url) - negotiated_capabilities = self._negotiate_receive_pack_capabilities( - server_capabilities) + negotiated_capabilities, agent = ( + self._negotiate_receive_pack_capabilities(server_capabilities)) negotiated_capabilities.add(capability_agent()) if CAPABILITY_REPORT_STATUS in negotiated_capabilities: @@ -1697,27 +1738,28 @@ def send_pack(self, path, update_refs, generate_pack_data, new_refs = update_refs(dict(old_refs)) if new_refs is None: # Determine wants function is aborting the push. - return old_refs + return SendPackResult(old_refs, agent=agent, ref_status={}) + if set(new_refs.items()).issubset(set(old_refs.items())): + return SendPackResult(new_refs, agent=agent, ref_status={}) if self.dumb: raise NotImplementedError(self.fetch_pack) req_data = BytesIO() req_proto = Protocol(None, req_data.write) (have, want) = self._handle_receive_pack_head( req_proto, negotiated_capabilities, old_refs, new_refs) - if not want and set(new_refs.items()).issubset(set(old_refs.items())): - return new_refs pack_data_count, pack_data = generate_pack_data( have, want, ofs_delta=(CAPABILITY_OFS_DELTA in negotiated_capabilities)) - if pack_data_count: + if self._should_send_pack(new_refs): write_pack_data(req_proto.write_file(), pack_data_count, pack_data) resp, read = self._smart_request("git-receive-pack", url, data=req_data.getvalue()) try: resp_proto = Protocol(read, None) - self._handle_receive_pack_tail( + ref_status = self._handle_receive_pack_tail( resp_proto, negotiated_capabilities, progress) - return new_refs + return SendPackResult( + new_refs, agent=agent, ref_status=ref_status) finally: resp.close() @@ -1749,7 +1791,7 @@ def fetch_pack(self, path, determine_wants, graph_walker, pack_data, if not wants: return FetchPackResult(refs, symrefs, agent) if self.dumb: - raise NotImplementedError(self.send_pack) + raise NotImplementedError(self.fetch_pack) req_data = BytesIO() req_proto = Protocol(None, req_data.write) (new_shallow, new_unshallow) = self._handle_upload_pack_head( @@ -1793,7 +1835,7 @@ def get_transport_and_path_from_url(url, config=None, **kwargs): Tuple with client instance and relative path. """ - parsed = urlparse.urlparse(url) + parsed = urlparse(url) if parsed.scheme == 'git': return (TCPGitClient.from_parsedurl(parsed, **kwargs), parsed.path) @@ -1861,3 +1903,25 @@ def get_transport_and_path(location, **kwargs): return default_local_git_client_cls(**kwargs), location else: return SSHGitClient(hostname, username=username, **kwargs), path + + +DEFAULT_GIT_CREDENTIALS_PATHS = [ + os.path.expanduser('~/.git-credentials'), + get_xdg_config_home_path('git', 'credentials')] + + +def get_credentials_from_store(scheme, hostname, username=None, + fnames=DEFAULT_GIT_CREDENTIALS_PATHS): + for fname in fnames: + try: + with open(fname, 'rb') as f: + for line in f: + parsed_line = urlparse(line) + if (parsed_line.scheme == scheme and + parsed_line.hostname == hostname and + (username is None or + parsed_line.username == username)): + return parsed_line.username, parsed_line.password + except FileNotFoundError: + # If the file doesn't exist, try the next one. + continue diff --git a/ext_libs/dulwich/config.py b/ext_libs/dulwich/config.py index d19038f3..57cf41a9 100644 --- a/ext_libs/dulwich/config.py +++ b/ext_libs/dulwich/config.py @@ -26,10 +26,11 @@ subsections """ -import errno import os import sys +from typing import BinaryIO, Tuple, Optional + from collections import ( OrderedDict, ) @@ -381,12 +382,17 @@ class ConfigFile(ConfigDict): """A Git configuration file, like .git/config or ~/.gitconfig. """ + def __init__(self, values=None, encoding=None): + super(ConfigFile, self).__init__(values=values, encoding=encoding) + self.path = None + @classmethod - def from_file(cls, f): + def from_file(cls, f: BinaryIO) -> 'ConfigFile': """Read configuration from a file-like object.""" ret = cls() - section = None + section = None # type: Optional[Tuple[bytes, ...]] setting = None + continuation = None for lineno, line in enumerate(f.readlines()): line = line.lstrip() if setting is None: @@ -430,7 +436,7 @@ def from_file(cls, f): value = b"true" setting = setting.strip() if not _check_variable_name(setting): - raise ValueError("invalid variable name %s" % setting) + raise ValueError("invalid variable name %r" % setting) if value.endswith(b"\\\n"): continuation = value[:-2] else: @@ -450,21 +456,21 @@ def from_file(cls, f): return ret @classmethod - def from_path(cls, path): + def from_path(cls, path) -> 'ConfigFile': """Read configuration from a file on disk.""" with GitFile(path, 'rb') as f: ret = cls.from_file(f) ret.path = path return ret - def write_to_path(self, path=None): + def write_to_path(self, path=None) -> None: """Write configuration to a file on disk.""" if path is None: path = self.path with GitFile(path, 'wb') as f: self.write_to_file(f) - def write_to_file(self, f): + def write_to_file(self, f: BinaryIO) -> None: """Write configuration to a file-like object.""" for section, values in self._values.items(): try: @@ -487,6 +493,13 @@ def write_to_file(self, f): f.write(b"\t" + key + b" = " + value + b"\n") +def get_xdg_config_home_path(*path_segments): + xdg_config_home = os.environ.get( + "XDG_CONFIG_HOME", os.path.expanduser("~/.config/"), + ) + return os.path.join(xdg_config_home, *path_segments) + + class StackedConfig(Config): """Configuration which reads from multiple config files..""" @@ -509,11 +522,7 @@ def default_backends(cls): """ paths = [] paths.append(os.path.expanduser("~/.gitconfig")) - - xdg_config_home = os.environ.get( - "XDG_CONFIG_HOME", os.path.expanduser("~/.config/"), - ) - paths.append(os.path.join(xdg_config_home, "git", "config")) + paths.append(get_xdg_config_home_path("git", "config")) if "GIT_CONFIG_NOSYSTEM" not in os.environ: paths.append("/etc/gitconfig") @@ -522,11 +531,8 @@ def default_backends(cls): for path in paths: try: cf = ConfigFile.from_path(path) - except (IOError, OSError) as e: - if e.errno != errno.ENOENT: - raise - else: - continue + except FileNotFoundError: + continue backends.append(cf) return backends diff --git a/ext_libs/dulwich/contrib/README.md b/ext_libs/dulwich/contrib/README.md deleted file mode 100644 index 3e65c012..00000000 --- a/ext_libs/dulwich/contrib/README.md +++ /dev/null @@ -1,3 +0,0 @@ -This directory contains code that some may find useful. Code here is not an official -part of Dulwich, and may no longer work. Unlike the rest of Dulwich, it is not regularly -tested. diff --git a/ext_libs/dulwich/contrib/__init__.py b/ext_libs/dulwich/contrib/__init__.py deleted file mode 100644 index 27689ec9..00000000 --- a/ext_libs/dulwich/contrib/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# __init__.py -- Contrib module for Dulwich -# Copyright (C) 2014 Jelmer Vernooij -# -# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU -# General Public License as public by the Free Software Foundation; version 2.0 -# or (at your option) any later version. You can redistribute it and/or -# modify it under the terms of either of these two licenses. -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# You should have received a copy of the licenses; if not, see -# for a copy of the GNU General Public License -# and for a copy of the Apache -# License, Version 2.0. -# - - -def test_suite(): - import unittest - names = [ - 'release_robot', - 'swift', - ] - module_names = ['dulwich.contrib.test_' + name for name in names] - loader = unittest.TestLoader() - return loader.loadTestsFromNames(module_names) diff --git a/ext_libs/dulwich/contrib/paramiko_vendor.py b/ext_libs/dulwich/contrib/paramiko_vendor.py deleted file mode 100644 index bfd1715b..00000000 --- a/ext_libs/dulwich/contrib/paramiko_vendor.py +++ /dev/null @@ -1,111 +0,0 @@ -# paramiko_vendor.py -- paramiko implementation of the SSHVendor interface -# Copyright (C) 2013 Aaron O'Mullan -# -# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU -# General Public License as public by the Free Software Foundation; version 2.0 -# or (at your option) any later version. You can redistribute it and/or -# modify it under the terms of either of these two licenses. -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# You should have received a copy of the licenses; if not, see -# for a copy of the GNU General Public License -# and for a copy of the Apache -# License, Version 2.0. -# - -"""Paramiko SSH support for Dulwich. - -To use this implementation as the SSH implementation in Dulwich, override -the dulwich.client.get_ssh_vendor attribute: - - >>> from dulwich import client as _mod_client - >>> from dulwich.contrib.paramiko_vendor import ParamikoSSHVendor - >>> _mod_client.get_ssh_vendor = ParamikoSSHVendor - -This implementation is experimental and does not have any tests. -""" - -import paramiko -import paramiko.client - - -class _ParamikoWrapper(object): - - def __init__(self, client, channel): - self.client = client - self.channel = channel - - # Channel must block - self.channel.setblocking(True) - - @property - def stderr(self): - return self.channel.makefile_stderr() - - def can_read(self): - return self.channel.recv_ready() - - def write(self, data): - return self.channel.sendall(data) - - def read(self, n=None): - data = self.channel.recv(n) - data_len = len(data) - - # Closed socket - if not data: - return b'' - - # Read more if needed - if n and data_len < n: - diff_len = n - data_len - return data + self.read(diff_len) - return data - - def close(self): - self.channel.close() - - -class ParamikoSSHVendor(object): - # http://docs.paramiko.org/en/2.4/api/client.html - - def __init__(self, **kwargs): - self.kwargs = kwargs - - def run_command(self, host, command, - username=None, port=None, - password=None, pkey=None, - key_filename=None, **kwargs): - - client = paramiko.SSHClient() - - connection_kwargs = {'hostname': host} - connection_kwargs.update(self.kwargs) - if username: - connection_kwargs['username'] = username - if port: - connection_kwargs['port'] = port - if password: - connection_kwargs['password'] = password - if pkey: - connection_kwargs['pkey'] = pkey - if key_filename: - connection_kwargs['key_filename'] = key_filename - connection_kwargs.update(kwargs) - - policy = paramiko.client.MissingHostKeyPolicy() - client.set_missing_host_key_policy(policy) - client.connect(**connection_kwargs) - - # Open SSH session - channel = client.get_transport().open_session() - - # Run commands - channel.exec_command(command) - - return _ParamikoWrapper(client, channel) diff --git a/ext_libs/dulwich/contrib/release_robot.py b/ext_libs/dulwich/contrib/release_robot.py deleted file mode 100644 index f2f280e0..00000000 --- a/ext_libs/dulwich/contrib/release_robot.py +++ /dev/null @@ -1,147 +0,0 @@ -# release_robot.py -# -# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU -# General Public License as public by the Free Software Foundation; version 2.0 -# or (at your option) any later version. You can redistribute it and/or -# modify it under the terms of either of these two licenses. -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# You should have received a copy of the licenses; if not, see -# for a copy of the GNU General Public License -# and for a copy of the Apache -# License, Version 2.0. -# - -"""Determine last version string from tags. - -Alternate to `Versioneer `_ using -`Dulwich `_ to sort tags by time from -newest to oldest. - -Copy the following into the package ``__init__.py`` module:: - - from dulwich.contrib.release_robot import get_current_version - __version__ = get_current_version() - -This example assumes the tags have a leading "v" like "v0.3", and that the -``.git`` folder is in a project folder that containts the package folder. - -EG:: - - * project - | - * .git - | - +-* package - | - * __init__.py <-- put __version__ here - - -""" - -import datetime -import re -import sys -import time - -from dulwich.repo import Repo - -# CONSTANTS -PROJDIR = '.' -PATTERN = r'[ a-zA-Z_\-]*([\d\.]+[\-\w\.]*)' - - -def get_recent_tags(projdir=PROJDIR): - """Get list of tags in order from newest to oldest and their datetimes. - - Args: - projdir: path to ``.git`` - Returns: - list of tags sorted by commit time from newest to oldest - - Each tag in the list contains the tag name, commit time, commit id, author - and any tag meta. If a tag isn't annotated, then its tag meta is ``None``. - Otherwise the tag meta is a tuple containing the tag time, tag id and tag - name. Time is in UTC. - """ - with Repo(projdir) as project: # dulwich repository object - refs = project.get_refs() # dictionary of refs and their SHA-1 values - tags = {} # empty dictionary to hold tags, commits and datetimes - # iterate over refs in repository - for key, value in refs.items(): - key = key.decode('utf-8') # compatible with Python-3 - obj = project.get_object(value) # dulwich object from SHA-1 - # don't just check if object is "tag" b/c it could be a "commit" - # instead check if "tags" is in the ref-name - if u'tags' not in key: - # skip ref if not a tag - continue - # strip the leading text from refs to get "tag name" - _, tag = key.rsplit(u'/', 1) - # check if tag object is "commit" or "tag" pointing to a "commit" - try: - commit = obj.object # a tuple (commit class, commit id) - except AttributeError: - commit = obj - tag_meta = None - else: - tag_meta = ( - datetime.datetime(*time.gmtime(obj.tag_time)[:6]), - obj.id.decode('utf-8'), - obj.name.decode('utf-8') - ) # compatible with Python-3 - commit = project.get_object(commit[1]) # commit object - # get tag commit datetime, but dulwich returns seconds since - # beginning of epoch, so use Python time module to convert it to - # timetuple then convert to datetime - tags[tag] = [ - datetime.datetime(*time.gmtime(commit.commit_time)[:6]), - commit.id.decode('utf-8'), - commit.author.decode('utf-8'), - tag_meta - ] # compatible with Python-3 - - # return list of tags sorted by their datetimes from newest to oldest - return sorted(tags.items(), key=lambda tag: tag[1][0], reverse=True) - - -def get_current_version(projdir=PROJDIR, pattern=PATTERN, logger=None): - """Return the most recent tag, using an options regular expression pattern. - - The default pattern will strip any characters preceding the first semantic - version. *EG*: "Release-0.2.1-rc.1" will be come "0.2.1-rc.1". If no match - is found, then the most recent tag is return without modification. - - Args: - projdir: path to ``.git`` - pattern: regular expression pattern with group that matches version - logger: a Python logging instance to capture exception - Returns: - tag matching first group in regular expression pattern - """ - tags = get_recent_tags(projdir) - try: - tag = tags[0][0] - except IndexError: - return - matches = re.match(pattern, tag) - try: - current_version = matches.group(1) - except (IndexError, AttributeError) as err: - if logger: - logger.exception(err) - return tag - return current_version - - -if __name__ == '__main__': - if len(sys.argv) > 1: - _PROJDIR = sys.argv[1] - else: - _PROJDIR = PROJDIR - print(get_current_version(projdir=_PROJDIR)) diff --git a/ext_libs/dulwich/contrib/swift.py b/ext_libs/dulwich/contrib/swift.py deleted file mode 100644 index eb96d2e9..00000000 --- a/ext_libs/dulwich/contrib/swift.py +++ /dev/null @@ -1,1058 +0,0 @@ -# swift.py -- Repo implementation atop OpenStack SWIFT -# Copyright (C) 2013 eNovance SAS -# -# Author: Fabien Boucher -# -# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU -# General Public License as public by the Free Software Foundation; version 2.0 -# or (at your option) any later version. You can redistribute it and/or -# modify it under the terms of either of these two licenses. -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# You should have received a copy of the licenses; if not, see -# for a copy of the GNU General Public License -# and for a copy of the Apache -# License, Version 2.0. -# - -"""Repo implementation atop OpenStack SWIFT.""" - -# TODO: Refactor to share more code with dulwich/repo.py. -# TODO(fbo): Second attempt to _send() must be notified via real log -# TODO(fbo): More logs for operations - -import os -import stat -import zlib -import tempfile -import posixpath - -try: - import urlparse -except ImportError: - import urllib.parse as urlparse - -from io import BytesIO -try: - from ConfigParser import ConfigParser -except ImportError: - from configparser import ConfigParser -from geventhttpclient import HTTPClient - -from dulwich.greenthreads import ( - GreenThreadsMissingObjectFinder, - GreenThreadsObjectStoreIterator, - ) - -from dulwich.lru_cache import LRUSizeCache -from dulwich.objects import ( - Blob, - Commit, - Tree, - Tag, - S_ISGITLINK, - ) -from dulwich.object_store import ( - PackBasedObjectStore, - PACKDIR, - INFODIR, - ) -from dulwich.pack import ( - PackData, - Pack, - PackIndexer, - PackStreamCopier, - write_pack_header, - compute_file_sha, - iter_sha1, - write_pack_index_v2, - load_pack_index_file, - read_pack_header, - _compute_object_size, - unpack_object, - write_pack_object, - ) -from dulwich.protocol import TCP_GIT_PORT -from dulwich.refs import ( - InfoRefsContainer, - read_info_refs, - write_info_refs, - ) -from dulwich.repo import ( - BaseRepo, - OBJECTDIR, - ) -from dulwich.server import ( - Backend, - TCPGitServer, - ) - -try: - from simplejson import loads as json_loads - from simplejson import dumps as json_dumps -except ImportError: - from json import loads as json_loads - from json import dumps as json_dumps - -import sys - - -""" -# Configuration file sample -[swift] -# Authentication URL (Keystone or Swift) -auth_url = http://127.0.0.1:5000/v2.0 -# Authentication version to use -auth_ver = 2 -# The tenant and username separated by a semicolon -username = admin;admin -# The user password -password = pass -# The Object storage region to use (auth v2) (Default RegionOne) -region_name = RegionOne -# The Object storage endpoint URL to use (auth v2) (Default internalURL) -endpoint_type = internalURL -# Concurrency to use for parallel tasks (Default 10) -concurrency = 10 -# Size of the HTTP pool (Default 10) -http_pool_length = 10 -# Timeout delay for HTTP connections (Default 20) -http_timeout = 20 -# Chunk size to read from pack (Bytes) (Default 12228) -chunk_length = 12228 -# Cache size (MBytes) (Default 20) -cache_length = 20 -""" - - -class PackInfoObjectStoreIterator(GreenThreadsObjectStoreIterator): - - def __len__(self): - while len(self.finder.objects_to_send): - for _ in range(0, len(self.finder.objects_to_send)): - sha = self.finder.next() - self._shas.append(sha) - return len(self._shas) - - -class PackInfoMissingObjectFinder(GreenThreadsMissingObjectFinder): - - def next(self): - while True: - if not self.objects_to_send: - return None - (sha, name, leaf) = self.objects_to_send.pop() - if sha not in self.sha_done: - break - if not leaf: - info = self.object_store.pack_info_get(sha) - if info[0] == Commit.type_num: - self.add_todo([(info[2], "", False)]) - elif info[0] == Tree.type_num: - self.add_todo([tuple(i) for i in info[1]]) - elif info[0] == Tag.type_num: - self.add_todo([(info[1], None, False)]) - if sha in self._tagged: - self.add_todo([(self._tagged[sha], None, True)]) - self.sha_done.add(sha) - self.progress("counting objects: %d\r" % len(self.sha_done)) - return (sha, name) - - -def load_conf(path=None, file=None): - """Load configuration in global var CONF - - Args: - path: The path to the configuration file - file: If provided read instead the file like object - """ - conf = ConfigParser() - if file: - try: - conf.read_file(file, path) - except AttributeError: - # read_file only exists in Python3 - conf.readfp(file) - return conf - confpath = None - if not path: - try: - confpath = os.environ['DULWICH_SWIFT_CFG'] - except KeyError: - raise Exception("You need to specify a configuration file") - else: - confpath = path - if not os.path.isfile(confpath): - raise Exception("Unable to read configuration file %s" % confpath) - conf.read(confpath) - return conf - - -def swift_load_pack_index(scon, filename): - """Read a pack index file from Swift - - Args: - scon: a `SwiftConnector` instance - filename: Path to the index file objectise - Returns: a `PackIndexer` instance - """ - with scon.get_object(filename) as f: - return load_pack_index_file(filename, f) - - -def pack_info_create(pack_data, pack_index): - pack = Pack.from_objects(pack_data, pack_index) - info = {} - for obj in pack.iterobjects(): - # Commit - if obj.type_num == Commit.type_num: - info[obj.id] = (obj.type_num, obj.parents, obj.tree) - # Tree - elif obj.type_num == Tree.type_num: - shas = [(s, n, not stat.S_ISDIR(m)) for - n, m, s in obj.items() if not S_ISGITLINK(m)] - info[obj.id] = (obj.type_num, shas) - # Blob - elif obj.type_num == Blob.type_num: - info[obj.id] = None - # Tag - elif obj.type_num == Tag.type_num: - info[obj.id] = (obj.type_num, obj.object[1]) - return zlib.compress(json_dumps(info)) - - -def load_pack_info(filename, scon=None, file=None): - if not file: - f = scon.get_object(filename) - else: - f = file - if not f: - return None - try: - return json_loads(zlib.decompress(f.read())) - finally: - f.close() - - -class SwiftException(Exception): - pass - - -class SwiftConnector(object): - """A Connector to swift that manage authentication and errors catching - """ - - def __init__(self, root, conf): - """ Initialize a SwiftConnector - - Args: - root: The swift container that will act as Git bare repository - conf: A ConfigParser Object - """ - self.conf = conf - self.auth_ver = self.conf.get("swift", "auth_ver") - if self.auth_ver not in ["1", "2"]: - raise NotImplementedError( - "Wrong authentication version use either 1 or 2") - self.auth_url = self.conf.get("swift", "auth_url") - self.user = self.conf.get("swift", "username") - self.password = self.conf.get("swift", "password") - self.concurrency = self.conf.getint('swift', 'concurrency') or 10 - self.http_timeout = self.conf.getint('swift', 'http_timeout') or 20 - self.http_pool_length = \ - self.conf.getint('swift', 'http_pool_length') or 10 - self.region_name = self.conf.get("swift", "region_name") or "RegionOne" - self.endpoint_type = \ - self.conf.get("swift", "endpoint_type") or "internalURL" - self.cache_length = self.conf.getint("swift", "cache_length") or 20 - self.chunk_length = self.conf.getint("swift", "chunk_length") or 12228 - self.root = root - block_size = 1024 * 12 # 12KB - if self.auth_ver == "1": - self.storage_url, self.token = self.swift_auth_v1() - else: - self.storage_url, self.token = self.swift_auth_v2() - - token_header = {'X-Auth-Token': str(self.token)} - self.httpclient = \ - HTTPClient.from_url(str(self.storage_url), - concurrency=self.http_pool_length, - block_size=block_size, - connection_timeout=self.http_timeout, - network_timeout=self.http_timeout, - headers=token_header) - self.base_path = str(posixpath.join( - urlparse.urlparse(self.storage_url).path, self.root)) - - def swift_auth_v1(self): - self.user = self.user.replace(";", ":") - auth_httpclient = HTTPClient.from_url( - self.auth_url, - connection_timeout=self.http_timeout, - network_timeout=self.http_timeout, - ) - headers = {'X-Auth-User': self.user, - 'X-Auth-Key': self.password} - path = urlparse.urlparse(self.auth_url).path - - ret = auth_httpclient.request('GET', path, headers=headers) - - # Should do something with redirections (301 in my case) - - if ret.status_code < 200 or ret.status_code >= 300: - raise SwiftException('AUTH v1.0 request failed on ' + - '%s with error code %s (%s)' - % (str(auth_httpclient.get_base_url()) + - path, ret.status_code, - str(ret.items()))) - storage_url = ret['X-Storage-Url'] - token = ret['X-Auth-Token'] - return storage_url, token - - def swift_auth_v2(self): - self.tenant, self.user = self.user.split(';') - auth_dict = {} - auth_dict['auth'] = {'passwordCredentials': - { - 'username': self.user, - 'password': self.password, - }, - 'tenantName': self.tenant} - auth_json = json_dumps(auth_dict) - headers = {'Content-Type': 'application/json'} - auth_httpclient = HTTPClient.from_url( - self.auth_url, - connection_timeout=self.http_timeout, - network_timeout=self.http_timeout, - ) - path = urlparse.urlparse(self.auth_url).path - if not path.endswith('tokens'): - path = posixpath.join(path, 'tokens') - ret = auth_httpclient.request('POST', path, - body=auth_json, - headers=headers) - - if ret.status_code < 200 or ret.status_code >= 300: - raise SwiftException('AUTH v2.0 request failed on ' + - '%s with error code %s (%s)' - % (str(auth_httpclient.get_base_url()) + - path, ret.status_code, - str(ret.items()))) - auth_ret_json = json_loads(ret.read()) - token = auth_ret_json['access']['token']['id'] - catalogs = auth_ret_json['access']['serviceCatalog'] - object_store = [o_store for o_store in catalogs if - o_store['type'] == 'object-store'][0] - endpoints = object_store['endpoints'] - endpoint = [endp for endp in endpoints if - endp["region"] == self.region_name][0] - return endpoint[self.endpoint_type], token - - def test_root_exists(self): - """Check that Swift container exist - - Returns: True if exist or None it not - """ - ret = self.httpclient.request('HEAD', self.base_path) - if ret.status_code == 404: - return None - if ret.status_code < 200 or ret.status_code > 300: - raise SwiftException('HEAD request failed with error code %s' - % ret.status_code) - return True - - def create_root(self): - """Create the Swift container - - Raises: - SwiftException: if unable to create - """ - if not self.test_root_exists(): - ret = self.httpclient.request('PUT', self.base_path) - if ret.status_code < 200 or ret.status_code > 300: - raise SwiftException('PUT request failed with error code %s' - % ret.status_code) - - def get_container_objects(self): - """Retrieve objects list in a container - - Returns: A list of dict that describe objects - or None if container does not exist - """ - qs = '?format=json' - path = self.base_path + qs - ret = self.httpclient.request('GET', path) - if ret.status_code == 404: - return None - if ret.status_code < 200 or ret.status_code > 300: - raise SwiftException('GET request failed with error code %s' - % ret.status_code) - content = ret.read() - return json_loads(content) - - def get_object_stat(self, name): - """Retrieve object stat - - Args: - name: The object name - Returns: - A dict that describe the object or None if object does not exist - """ - path = self.base_path + '/' + name - ret = self.httpclient.request('HEAD', path) - if ret.status_code == 404: - return None - if ret.status_code < 200 or ret.status_code > 300: - raise SwiftException('HEAD request failed with error code %s' - % ret.status_code) - resp_headers = {} - for header, value in ret.items(): - resp_headers[header.lower()] = value - return resp_headers - - def put_object(self, name, content): - """Put an object - - Args: - name: The object name - content: A file object - Raises: - SwiftException: if unable to create - """ - content.seek(0) - data = content.read() - path = self.base_path + '/' + name - headers = {'Content-Length': str(len(data))} - - def _send(): - ret = self.httpclient.request('PUT', path, - body=data, - headers=headers) - return ret - - try: - # Sometime got Broken Pipe - Dirty workaround - ret = _send() - except Exception: - # Second attempt work - ret = _send() - - if ret.status_code < 200 or ret.status_code > 300: - raise SwiftException('PUT request failed with error code %s' - % ret.status_code) - - def get_object(self, name, range=None): - """Retrieve an object - - Args: - name: The object name - range: A string range like "0-10" to - retrieve specified bytes in object content - Returns: - A file like instance or bytestring if range is specified - """ - headers = {} - if range: - headers['Range'] = 'bytes=%s' % range - path = self.base_path + '/' + name - ret = self.httpclient.request('GET', path, headers=headers) - if ret.status_code == 404: - return None - if ret.status_code < 200 or ret.status_code > 300: - raise SwiftException('GET request failed with error code %s' - % ret.status_code) - content = ret.read() - - if range: - return content - return BytesIO(content) - - def del_object(self, name): - """Delete an object - - Args: - name: The object name - Raises: - SwiftException: if unable to delete - """ - path = self.base_path + '/' + name - ret = self.httpclient.request('DELETE', path) - if ret.status_code < 200 or ret.status_code > 300: - raise SwiftException('DELETE request failed with error code %s' - % ret.status_code) - - def del_root(self): - """Delete the root container by removing container content - - Raises: - SwiftException: if unable to delete - """ - for obj in self.get_container_objects(): - self.del_object(obj['name']) - ret = self.httpclient.request('DELETE', self.base_path) - if ret.status_code < 200 or ret.status_code > 300: - raise SwiftException('DELETE request failed with error code %s' - % ret.status_code) - - -class SwiftPackReader(object): - """A SwiftPackReader that mimic read and sync method - - The reader allows to read a specified amount of bytes from - a given offset of a Swift object. A read offset is kept internaly. - The reader will read from Swift a specified amount of data to complete - its internal buffer. chunk_length specifiy the amount of data - to read from Swift. - """ - - def __init__(self, scon, filename, pack_length): - """Initialize a SwiftPackReader - - Args: - scon: a `SwiftConnector` instance - filename: the pack filename - pack_length: The size of the pack object - """ - self.scon = scon - self.filename = filename - self.pack_length = pack_length - self.offset = 0 - self.base_offset = 0 - self.buff = b'' - self.buff_length = self.scon.chunk_length - - def _read(self, more=False): - if more: - self.buff_length = self.buff_length * 2 - offset = self.base_offset - r = min(self.base_offset + self.buff_length, self.pack_length) - ret = self.scon.get_object(self.filename, range="%s-%s" % (offset, r)) - self.buff = ret - - def read(self, length): - """Read a specified amount of Bytes form the pack object - - Args: - length: amount of bytes to read - Returns: - a bytestring - """ - end = self.offset+length - if self.base_offset + end > self.pack_length: - data = self.buff[self.offset:] - self.offset = end - return data - if end > len(self.buff): - # Need to read more from swift - self._read(more=True) - return self.read(length) - data = self.buff[self.offset:end] - self.offset = end - return data - - def seek(self, offset): - """Seek to a specified offset - - Args: - offset: the offset to seek to - """ - self.base_offset = offset - self._read() - self.offset = 0 - - def read_checksum(self): - """Read the checksum from the pack - - Returns: the checksum bytestring - """ - return self.scon.get_object(self.filename, range="-20") - - -class SwiftPackData(PackData): - """The data contained in a packfile. - - We use the SwiftPackReader to read bytes from packs stored in Swift - using the Range header feature of Swift. - """ - - def __init__(self, scon, filename): - """ Initialize a SwiftPackReader - - Args: - scon: a `SwiftConnector` instance - filename: the pack filename - """ - self.scon = scon - self._filename = filename - self._header_size = 12 - headers = self.scon.get_object_stat(self._filename) - self.pack_length = int(headers['content-length']) - pack_reader = SwiftPackReader(self.scon, self._filename, - self.pack_length) - (version, self._num_objects) = read_pack_header(pack_reader.read) - self._offset_cache = LRUSizeCache(1024*1024*self.scon.cache_length, - compute_size=_compute_object_size) - self.pack = None - - def get_object_at(self, offset): - if offset in self._offset_cache: - return self._offset_cache[offset] - assert offset >= self._header_size - pack_reader = SwiftPackReader(self.scon, self._filename, - self.pack_length) - pack_reader.seek(offset) - unpacked, _ = unpack_object(pack_reader.read) - return (unpacked.pack_type_num, unpacked._obj()) - - def get_stored_checksum(self): - pack_reader = SwiftPackReader(self.scon, self._filename, - self.pack_length) - return pack_reader.read_checksum() - - def close(self): - pass - - -class SwiftPack(Pack): - """A Git pack object. - - Same implementation as pack.Pack except that _idx_load and - _data_load are bounded to Swift version of load_pack_index and - PackData. - """ - - def __init__(self, *args, **kwargs): - self.scon = kwargs['scon'] - del kwargs['scon'] - super(SwiftPack, self).__init__(*args, **kwargs) - self._pack_info_path = self._basename + '.info' - self._pack_info = None - self._pack_info_load = lambda: load_pack_info(self._pack_info_path, - self.scon) - self._idx_load = lambda: swift_load_pack_index(self.scon, - self._idx_path) - self._data_load = lambda: SwiftPackData(self.scon, self._data_path) - - @property - def pack_info(self): - """The pack data object being used.""" - if self._pack_info is None: - self._pack_info = self._pack_info_load() - return self._pack_info - - -class SwiftObjectStore(PackBasedObjectStore): - """A Swift Object Store - - Allow to manage a bare Git repository from Openstack Swift. - This object store only supports pack files and not loose objects. - """ - def __init__(self, scon): - """Open a Swift object store. - - Args: - scon: A `SwiftConnector` instance - """ - super(SwiftObjectStore, self).__init__() - self.scon = scon - self.root = self.scon.root - self.pack_dir = posixpath.join(OBJECTDIR, PACKDIR) - self._alternates = None - - def _update_pack_cache(self): - objects = self.scon.get_container_objects() - pack_files = [o['name'].replace(".pack", "") - for o in objects if o['name'].endswith(".pack")] - ret = [] - for basename in pack_files: - pack = SwiftPack(basename, scon=self.scon) - self._pack_cache[basename] = pack - ret.append(pack) - return ret - - def _iter_loose_objects(self): - """Loose objects are not supported by this repository - """ - return [] - - def iter_shas(self, finder): - """An iterator over pack's ObjectStore. - - Returns: a `ObjectStoreIterator` or `GreenThreadsObjectStoreIterator` - instance if gevent is enabled - """ - shas = iter(finder.next, None) - return PackInfoObjectStoreIterator( - self, shas, finder, self.scon.concurrency) - - def find_missing_objects(self, *args, **kwargs): - kwargs['concurrency'] = self.scon.concurrency - return PackInfoMissingObjectFinder(self, *args, **kwargs) - - def pack_info_get(self, sha): - for pack in self.packs: - if sha in pack: - return pack.pack_info[sha] - - def _collect_ancestors(self, heads, common=set()): - def _find_parents(commit): - for pack in self.packs: - if commit in pack: - try: - parents = pack.pack_info[commit][1] - except KeyError: - # Seems to have no parents - return [] - return parents - - bases = set() - commits = set() - queue = [] - queue.extend(heads) - while queue: - e = queue.pop(0) - if e in common: - bases.add(e) - elif e not in commits: - commits.add(e) - parents = _find_parents(e) - queue.extend(parents) - return (commits, bases) - - def add_pack(self): - """Add a new pack to this object store. - - Returns: Fileobject to write to and a commit function to - call when the pack is finished. - """ - f = BytesIO() - - def commit(): - f.seek(0) - pack = PackData(file=f, filename="") - entries = pack.sorted_entries() - if len(entries): - basename = posixpath.join(self.pack_dir, - "pack-%s" % - iter_sha1(entry[0] for - entry in entries)) - index = BytesIO() - write_pack_index_v2(index, entries, pack.get_stored_checksum()) - self.scon.put_object(basename + ".pack", f) - f.close() - self.scon.put_object(basename + ".idx", index) - index.close() - final_pack = SwiftPack(basename, scon=self.scon) - final_pack.check_length_and_checksum() - self._add_cached_pack(basename, final_pack) - return final_pack - else: - return None - - def abort(): - pass - return f, commit, abort - - def add_object(self, obj): - self.add_objects([(obj, None), ]) - - def _pack_cache_stale(self): - return False - - def _get_loose_object(self, sha): - return None - - def add_thin_pack(self, read_all, read_some): - """Read a thin pack - - Read it from a stream and complete it in a temporary file. - Then the pack and the corresponding index file are uploaded to Swift. - """ - fd, path = tempfile.mkstemp(prefix='tmp_pack_') - f = os.fdopen(fd, 'w+b') - try: - indexer = PackIndexer(f, resolve_ext_ref=self.get_raw) - copier = PackStreamCopier(read_all, read_some, f, - delta_iter=indexer) - copier.verify() - return self._complete_thin_pack(f, path, copier, indexer) - finally: - f.close() - os.unlink(path) - - def _complete_thin_pack(self, f, path, copier, indexer): - entries = list(indexer) - - # Update the header with the new number of objects. - f.seek(0) - write_pack_header(f, len(entries) + len(indexer.ext_refs())) - - # Must flush before reading (http://bugs.python.org/issue3207) - f.flush() - - # Rescan the rest of the pack, computing the SHA with the new header. - new_sha = compute_file_sha(f, end_ofs=-20) - - # Must reposition before writing (http://bugs.python.org/issue3207) - f.seek(0, os.SEEK_CUR) - - # Complete the pack. - for ext_sha in indexer.ext_refs(): - assert len(ext_sha) == 20 - type_num, data = self.get_raw(ext_sha) - offset = f.tell() - crc32 = write_pack_object(f, type_num, data, sha=new_sha) - entries.append((ext_sha, offset, crc32)) - pack_sha = new_sha.digest() - f.write(pack_sha) - f.flush() - - # Move the pack in. - entries.sort() - pack_base_name = posixpath.join( - self.pack_dir, - 'pack-' + iter_sha1(e[0] for e in entries).decode( - sys.getfilesystemencoding())) - self.scon.put_object(pack_base_name + '.pack', f) - - # Write the index. - filename = pack_base_name + '.idx' - index_file = BytesIO() - write_pack_index_v2(index_file, entries, pack_sha) - self.scon.put_object(filename, index_file) - - # Write pack info. - f.seek(0) - pack_data = PackData(filename="", file=f) - index_file.seek(0) - pack_index = load_pack_index_file('', index_file) - serialized_pack_info = pack_info_create(pack_data, pack_index) - f.close() - index_file.close() - pack_info_file = BytesIO(serialized_pack_info) - filename = pack_base_name + '.info' - self.scon.put_object(filename, pack_info_file) - pack_info_file.close() - - # Add the pack to the store and return it. - final_pack = SwiftPack(pack_base_name, scon=self.scon) - final_pack.check_length_and_checksum() - self._add_cached_pack(pack_base_name, final_pack) - return final_pack - - -class SwiftInfoRefsContainer(InfoRefsContainer): - """Manage references in info/refs object. - """ - - def __init__(self, scon, store): - self.scon = scon - self.filename = 'info/refs' - self.store = store - f = self.scon.get_object(self.filename) - if not f: - f = BytesIO(b'') - super(SwiftInfoRefsContainer, self).__init__(f) - - def _load_check_ref(self, name, old_ref): - self._check_refname(name) - f = self.scon.get_object(self.filename) - if not f: - return {} - refs = read_info_refs(f) - if old_ref is not None: - if refs[name] != old_ref: - return False - return refs - - def _write_refs(self, refs): - f = BytesIO() - f.writelines(write_info_refs(refs, self.store)) - self.scon.put_object(self.filename, f) - - def set_if_equals(self, name, old_ref, new_ref): - """Set a refname to new_ref only if it currently equals old_ref. - """ - if name == 'HEAD': - return True - refs = self._load_check_ref(name, old_ref) - if not isinstance(refs, dict): - return False - refs[name] = new_ref - self._write_refs(refs) - self._refs[name] = new_ref - return True - - def remove_if_equals(self, name, old_ref): - """Remove a refname only if it currently equals old_ref. - """ - if name == 'HEAD': - return True - refs = self._load_check_ref(name, old_ref) - if not isinstance(refs, dict): - return False - del refs[name] - self._write_refs(refs) - del self._refs[name] - return True - - def allkeys(self): - try: - self._refs['HEAD'] = self._refs['refs/heads/master'] - except KeyError: - pass - return self._refs.keys() - - -class SwiftRepo(BaseRepo): - - def __init__(self, root, conf): - """Init a Git bare Repository on top of a Swift container. - - References are managed in info/refs objects by - `SwiftInfoRefsContainer`. The root attribute is the Swift - container that contain the Git bare repository. - - Args: - root: The container which contains the bare repo - conf: A ConfigParser object - """ - self.root = root.lstrip('/') - self.conf = conf - self.scon = SwiftConnector(self.root, self.conf) - objects = self.scon.get_container_objects() - if not objects: - raise Exception('There is not any GIT repo here : %s' % self.root) - objects = [o['name'].split('/')[0] for o in objects] - if OBJECTDIR not in objects: - raise Exception('This repository (%s) is not bare.' % self.root) - self.bare = True - self._controldir = self.root - object_store = SwiftObjectStore(self.scon) - refs = SwiftInfoRefsContainer(self.scon, object_store) - BaseRepo.__init__(self, object_store, refs) - - def _determine_file_mode(self): - """Probe the file-system to determine whether permissions can be trusted. - - Returns: True if permissions can be trusted, False otherwise. - """ - return False - - def _put_named_file(self, filename, contents): - """Put an object in a Swift container - - Args: - filename: the path to the object to put on Swift - contents: the content as bytestring - """ - with BytesIO() as f: - f.write(contents) - self.scon.put_object(filename, f) - - @classmethod - def init_bare(cls, scon, conf): - """Create a new bare repository. - - Args: - scon: a `SwiftConnector` instance - conf: a ConfigParser object - Returns: - a `SwiftRepo` instance - """ - scon.create_root() - for obj in [posixpath.join(OBJECTDIR, PACKDIR), - posixpath.join(INFODIR, 'refs')]: - scon.put_object(obj, BytesIO(b'')) - ret = cls(scon.root, conf) - ret._init_files(True) - return ret - - -class SwiftSystemBackend(Backend): - - def __init__(self, logger, conf): - self.conf = conf - self.logger = logger - - def open_repository(self, path): - self.logger.info('opening repository at %s', path) - return SwiftRepo(path, self.conf) - - -def cmd_daemon(args): - """Entry point for starting a TCP git server.""" - import optparse - parser = optparse.OptionParser() - parser.add_option("-l", "--listen_address", dest="listen_address", - default="127.0.0.1", - help="Binding IP address.") - parser.add_option("-p", "--port", dest="port", type=int, - default=TCP_GIT_PORT, - help="Binding TCP port.") - parser.add_option("-c", "--swift_config", dest="swift_config", - default="", - help="Path to the configuration file for Swift backend.") - options, args = parser.parse_args(args) - - try: - import gevent - import geventhttpclient # noqa: F401 - except ImportError: - print("gevent and geventhttpclient libraries are mandatory " - " for use the Swift backend.") - sys.exit(1) - import gevent.monkey - gevent.monkey.patch_socket() - from dulwich import log_utils - logger = log_utils.getLogger(__name__) - conf = load_conf(options.swift_config) - backend = SwiftSystemBackend(logger, conf) - - log_utils.default_logging_config() - server = TCPGitServer(backend, options.listen_address, - port=options.port) - server.serve_forever() - - -def cmd_init(args): - import optparse - parser = optparse.OptionParser() - parser.add_option("-c", "--swift_config", dest="swift_config", - default="", - help="Path to the configuration file for Swift backend.") - options, args = parser.parse_args(args) - - conf = load_conf(options.swift_config) - if args == []: - parser.error("missing repository name") - repo = args[0] - scon = SwiftConnector(repo, conf) - SwiftRepo.init_bare(scon, conf) - - -def main(argv=sys.argv): - commands = { - "init": cmd_init, - "daemon": cmd_daemon, - } - - if len(sys.argv) < 2: - print("Usage: %s <%s> [OPTIONS...]" % ( - sys.argv[0], "|".join(commands.keys()))) - sys.exit(1) - - cmd = sys.argv[1] - if cmd not in commands: - print("No such subcommand: %s" % cmd) - sys.exit(1) - commands[cmd](sys.argv[2:]) - - -if __name__ == '__main__': - main() diff --git a/ext_libs/dulwich/contrib/test_release_robot.py b/ext_libs/dulwich/contrib/test_release_robot.py deleted file mode 100644 index 4bab3a18..00000000 --- a/ext_libs/dulwich/contrib/test_release_robot.py +++ /dev/null @@ -1,127 +0,0 @@ -# release_robot.py -# -# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU -# General Public License as public by the Free Software Foundation; version 2.0 -# or (at your option) any later version. You can redistribute it and/or -# modify it under the terms of either of these two licenses. -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# You should have received a copy of the licenses; if not, see -# for a copy of the GNU General Public License -# and for a copy of the Apache -# License, Version 2.0. -# - -"""Tests for release_robot.""" - -import datetime -import os -import re -import shutil -import tempfile -import time -import unittest - -from dulwich.contrib import release_robot -from dulwich.repo import Repo -from dulwich.tests.utils import make_commit, make_tag - -BASEDIR = os.path.abspath(os.path.dirname(__file__)) # this directory - - -def gmtime_to_datetime(gmt): - return datetime.datetime(*time.gmtime(gmt)[:6]) - - -class TagPatternTests(unittest.TestCase): - """test tag patterns""" - - def test_tag_pattern(self): - """test tag patterns""" - test_cases = { - '0.3': '0.3', 'v0.3': '0.3', 'release0.3': '0.3', - 'Release-0.3': '0.3', 'v0.3rc1': '0.3rc1', 'v0.3-rc1': '0.3-rc1', - 'v0.3-rc.1': '0.3-rc.1', 'version 0.3': '0.3', - 'version_0.3_rc_1': '0.3_rc_1', 'v1': '1', '0.3rc1': '0.3rc1' - } - for testcase, version in test_cases.items(): - matches = re.match(release_robot.PATTERN, testcase) - self.assertEqual(matches.group(1), version) - - -class GetRecentTagsTest(unittest.TestCase): - """test get recent tags""" - - # Git repo for dulwich project - test_repo = os.path.join(BASEDIR, 'dulwich_test_repo.zip') - committer = b"Mark Mikofski " - test_tags = [b'v0.1a', b'v0.1'] - tag_test_data = { - test_tags[0]: [1484788003, b'3' * 40, None], - test_tags[1]: [1484788314, b'1' * 40, (1484788401, b'2' * 40)] - } - - @classmethod - def setUpClass(cls): - cls.projdir = tempfile.mkdtemp() # temporary project directory - cls.repo = Repo.init(cls.projdir) # test repo - obj_store = cls.repo.object_store # test repo object store - # commit 1 ('2017-01-19T01:06:43') - cls.c1 = make_commit( - id=cls.tag_test_data[cls.test_tags[0]][1], - commit_time=cls.tag_test_data[cls.test_tags[0]][0], - message=b'unannotated tag', - author=cls.committer - ) - obj_store.add_object(cls.c1) - # tag 1: unannotated - cls.t1 = cls.test_tags[0] - cls.repo[b'refs/tags/' + cls.t1] = cls.c1.id # add unannotated tag - # commit 2 ('2017-01-19T01:11:54') - cls.c2 = make_commit( - id=cls.tag_test_data[cls.test_tags[1]][1], - commit_time=cls.tag_test_data[cls.test_tags[1]][0], - message=b'annotated tag', - parents=[cls.c1.id], - author=cls.committer - ) - obj_store.add_object(cls.c2) - # tag 2: annotated ('2017-01-19T01:13:21') - cls.t2 = make_tag( - cls.c2, - id=cls.tag_test_data[cls.test_tags[1]][2][1], - name=cls.test_tags[1], - tag_time=cls.tag_test_data[cls.test_tags[1]][2][0] - ) - obj_store.add_object(cls.t2) - cls.repo[b'refs/heads/master'] = cls.c2.id - cls.repo[b'refs/tags/' + cls.t2.name] = cls.t2.id # add annotated tag - - @classmethod - def tearDownClass(cls): - cls.repo.close() - shutil.rmtree(cls.projdir) - - def test_get_recent_tags(self): - """test get recent tags""" - tags = release_robot.get_recent_tags(self.projdir) # get test tags - for tag, metadata in tags: - tag = tag.encode('utf-8') - test_data = self.tag_test_data[tag] # test data tag - # test commit date, id and author name - self.assertEqual(metadata[0], gmtime_to_datetime(test_data[0])) - self.assertEqual(metadata[1].encode('utf-8'), test_data[1]) - self.assertEqual(metadata[2].encode('utf-8'), self.committer) - # skip unannotated tags - tag_obj = test_data[2] - if not tag_obj: - continue - # tag date, id and name - self.assertEqual(metadata[3][0], gmtime_to_datetime(tag_obj[0])) - self.assertEqual(metadata[3][1].encode('utf-8'), tag_obj[1]) - self.assertEqual(metadata[3][2].encode('utf-8'), tag) diff --git a/ext_libs/dulwich/contrib/test_swift.py b/ext_libs/dulwich/contrib/test_swift.py deleted file mode 100644 index 090d5343..00000000 --- a/ext_libs/dulwich/contrib/test_swift.py +++ /dev/null @@ -1,656 +0,0 @@ -# test_swift.py -- Unittests for the Swift backend. -# Copyright (C) 2013 eNovance SAS -# -# Author: Fabien Boucher -# -# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU -# General Public License as public by the Free Software Foundation; version 2.0 -# or (at your option) any later version. You can redistribute it and/or -# modify it under the terms of either of these two licenses. -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# You should have received a copy of the licenses; if not, see -# for a copy of the GNU General Public License -# and for a copy of the Apache -# License, Version 2.0. -# - -"""Tests for dulwich.contrib.swift.""" - -import posixpath - -from time import time -from io import BytesIO -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - -import sys -from unittest import skipIf - -from dulwich.tests import ( - TestCase, - ) -from dulwich.tests.test_object_store import ( - ObjectStoreTests, - ) -from dulwich.tests.utils import ( - build_pack, - ) -from dulwich.objects import ( - Blob, - Commit, - Tree, - Tag, - parse_timezone, - ) -from dulwich.pack import ( - REF_DELTA, - write_pack_index_v2, - PackData, - load_pack_index_file, - ) - -try: - from simplejson import dumps as json_dumps -except ImportError: - from json import dumps as json_dumps - -missing_libs = [] - -try: - import gevent # noqa:F401 -except ImportError: - missing_libs.append("gevent") - -try: - import geventhttpclient # noqa:F401 -except ImportError: - missing_libs.append("geventhttpclient") - -try: - from mock import patch -except ImportError: - missing_libs.append("mock") - -skipmsg = "Required libraries are not installed (%r)" % missing_libs - -skipIfPY3 = skipIf(sys.version_info[0] == 3, - "SWIFT module not yet ported to python3.") - -if not missing_libs: - from dulwich.contrib import swift - -config_file = """[swift] -auth_url = http://127.0.0.1:8080/auth/%(version_str)s -auth_ver = %(version_int)s -username = test;tester -password = testing -region_name = %(region_name)s -endpoint_type = %(endpoint_type)s -concurrency = %(concurrency)s -chunk_length = %(chunk_length)s -cache_length = %(cache_length)s -http_pool_length = %(http_pool_length)s -http_timeout = %(http_timeout)s -""" - -def_config_file = {'version_str': 'v1.0', - 'version_int': 1, - 'concurrency': 1, - 'chunk_length': 12228, - 'cache_length': 1, - 'region_name': 'test', - 'endpoint_type': 'internalURL', - 'http_pool_length': 1, - 'http_timeout': 1} - - -def create_swift_connector(store={}): - return lambda root, conf: FakeSwiftConnector(root, - conf=conf, - store=store) - - -class Response(object): - - def __init__(self, headers={}, status=200, content=None): - self.headers = headers - self.status_code = status - self.content = content - - def __getitem__(self, key): - return self.headers[key] - - def items(self): - return self.headers.items() - - def read(self): - return self.content - - -def fake_auth_request_v1(*args, **kwargs): - ret = Response({'X-Storage-Url': - 'http://127.0.0.1:8080/v1.0/AUTH_fakeuser', - 'X-Auth-Token': '12' * 10}, - 200) - return ret - - -def fake_auth_request_v1_error(*args, **kwargs): - ret = Response({}, - 401) - return ret - - -def fake_auth_request_v2(*args, **kwargs): - s_url = 'http://127.0.0.1:8080/v1.0/AUTH_fakeuser' - resp = {'access': {'token': {'id': '12' * 10}, - 'serviceCatalog': - [ - {'type': 'object-store', - 'endpoints': [{'region': 'test', - 'internalURL': s_url, - }, - ] - }, - ] - } - } - ret = Response(status=200, content=json_dumps(resp)) - return ret - - -def create_commit(data, marker=b'Default', blob=None): - if not blob: - blob = Blob.from_string(b'The blob content ' + marker) - tree = Tree() - tree.add(b"thefile_" + marker, 0o100644, blob.id) - cmt = Commit() - if data: - assert isinstance(data[-1], Commit) - cmt.parents = [data[-1].id] - cmt.tree = tree.id - author = b"John Doe " + marker + b" " - cmt.author = cmt.committer = author - tz = parse_timezone(b'-0200')[0] - cmt.commit_time = cmt.author_time = int(time()) - cmt.commit_timezone = cmt.author_timezone = tz - cmt.encoding = b"UTF-8" - cmt.message = b"The commit message " + marker - tag = Tag() - tag.tagger = b"john@doe.net" - tag.message = b"Annotated tag" - tag.tag_timezone = parse_timezone(b'-0200')[0] - tag.tag_time = cmt.author_time - tag.object = (Commit, cmt.id) - tag.name = b"v_" + marker + b"_0.1" - return blob, tree, tag, cmt - - -def create_commits(length=1, marker=b'Default'): - data = [] - for i in range(0, length): - _marker = ("%s_%s" % (marker, i)).encode() - blob, tree, tag, cmt = create_commit(data, _marker) - data.extend([blob, tree, tag, cmt]) - return data - - -@skipIf(missing_libs, skipmsg) -class FakeSwiftConnector(object): - - def __init__(self, root, conf, store=None): - if store: - self.store = store - else: - self.store = {} - self.conf = conf - self.root = root - self.concurrency = 1 - self.chunk_length = 12228 - self.cache_length = 1 - - def put_object(self, name, content): - name = posixpath.join(self.root, name) - if hasattr(content, 'seek'): - content.seek(0) - content = content.read() - self.store[name] = content - - def get_object(self, name, range=None): - name = posixpath.join(self.root, name) - if not range: - try: - return BytesIO(self.store[name]) - except KeyError: - return None - else: - l, r = range.split('-') - try: - if not l: - r = -int(r) - return self.store[name][r:] - else: - return self.store[name][int(l):int(r)] - except KeyError: - return None - - def get_container_objects(self): - return [{'name': k.replace(self.root + '/', '')} - for k in self.store] - - def create_root(self): - if self.root in self.store.keys(): - pass - else: - self.store[self.root] = '' - - def get_object_stat(self, name): - name = posixpath.join(self.root, name) - if name not in self.store: - return None - return {'content-length': len(self.store[name])} - - -@skipIf(missing_libs, skipmsg) -@skipIfPY3 -class TestSwiftObjectStore(TestCase): - - def setUp(self): - super(TestSwiftObjectStore, self).setUp() - self.conf = swift.load_conf(file=StringIO(config_file % - def_config_file)) - self.fsc = FakeSwiftConnector('fakerepo', conf=self.conf) - - def _put_pack(self, sos, commit_amount=1, marker='Default'): - odata = create_commits(length=commit_amount, marker=marker) - data = [(d.type_num, d.as_raw_string()) for d in odata] - f = BytesIO() - build_pack(f, data, store=sos) - sos.add_thin_pack(f.read, None) - return odata - - def test_load_packs(self): - store = {'fakerepo/objects/pack/pack-'+'1'*40+'.idx': '', - 'fakerepo/objects/pack/pack-'+'1'*40+'.pack': '', - 'fakerepo/objects/pack/pack-'+'1'*40+'.info': '', - 'fakerepo/objects/pack/pack-'+'2'*40+'.idx': '', - 'fakerepo/objects/pack/pack-'+'2'*40+'.pack': '', - 'fakerepo/objects/pack/pack-'+'2'*40+'.info': ''} - fsc = FakeSwiftConnector('fakerepo', conf=self.conf, store=store) - sos = swift.SwiftObjectStore(fsc) - packs = sos.packs - self.assertEqual(len(packs), 2) - for pack in packs: - self.assertTrue(isinstance(pack, swift.SwiftPack)) - - def test_add_thin_pack(self): - sos = swift.SwiftObjectStore(self.fsc) - self._put_pack(sos, 1, 'Default') - self.assertEqual(len(self.fsc.store), 3) - - def test_find_missing_objects(self): - commit_amount = 3 - sos = swift.SwiftObjectStore(self.fsc) - odata = self._put_pack(sos, commit_amount, 'Default') - head = odata[-1].id - i = sos.iter_shas(sos.find_missing_objects([], - [head, ], - progress=None, - get_tagged=None)) - self.assertEqual(len(i), commit_amount * 3) - shas = [d.id for d in odata] - for sha, path in i: - self.assertIn(sha.id, shas) - - def test_find_missing_objects_with_tag(self): - commit_amount = 3 - sos = swift.SwiftObjectStore(self.fsc) - odata = self._put_pack(sos, commit_amount, 'Default') - head = odata[-1].id - peeled_sha = dict([(sha.object[1], sha.id) - for sha in odata if isinstance(sha, Tag)]) - - def get_tagged(): - return peeled_sha - i = sos.iter_shas(sos.find_missing_objects([], - [head, ], - progress=None, - get_tagged=get_tagged)) - self.assertEqual(len(i), commit_amount * 4) - shas = [d.id for d in odata] - for sha, path in i: - self.assertIn(sha.id, shas) - - def test_find_missing_objects_with_common(self): - commit_amount = 3 - sos = swift.SwiftObjectStore(self.fsc) - odata = self._put_pack(sos, commit_amount, 'Default') - head = odata[-1].id - have = odata[7].id - i = sos.iter_shas(sos.find_missing_objects([have, ], - [head, ], - progress=None, - get_tagged=None)) - self.assertEqual(len(i), 3) - - def test_find_missing_objects_multiple_packs(self): - sos = swift.SwiftObjectStore(self.fsc) - commit_amount_a = 3 - odataa = self._put_pack(sos, commit_amount_a, 'Default1') - heada = odataa[-1].id - commit_amount_b = 2 - odatab = self._put_pack(sos, commit_amount_b, 'Default2') - headb = odatab[-1].id - i = sos.iter_shas(sos.find_missing_objects([], - [heada, headb], - progress=None, - get_tagged=None)) - self.assertEqual(len(self.fsc.store), 6) - self.assertEqual(len(i), - commit_amount_a * 3 + - commit_amount_b * 3) - shas = [d.id for d in odataa] - shas.extend([d.id for d in odatab]) - for sha, path in i: - self.assertIn(sha.id, shas) - - def test_add_thin_pack_ext_ref(self): - sos = swift.SwiftObjectStore(self.fsc) - odata = self._put_pack(sos, 1, 'Default1') - ref_blob_content = odata[0].as_raw_string() - ref_blob_id = odata[0].id - new_blob = Blob.from_string(ref_blob_content.replace('blob', - 'yummy blob')) - blob, tree, tag, cmt = \ - create_commit([], marker='Default2', blob=new_blob) - data = [(REF_DELTA, (ref_blob_id, blob.as_raw_string())), - (tree.type_num, tree.as_raw_string()), - (cmt.type_num, cmt.as_raw_string()), - (tag.type_num, tag.as_raw_string())] - f = BytesIO() - build_pack(f, data, store=sos) - sos.add_thin_pack(f.read, None) - self.assertEqual(len(self.fsc.store), 6) - - -@skipIf(missing_libs, skipmsg) -class TestSwiftRepo(TestCase): - - def setUp(self): - super(TestSwiftRepo, self).setUp() - self.conf = swift.load_conf(file=StringIO(config_file % - def_config_file)) - - def test_init(self): - store = {'fakerepo/objects/pack': ''} - with patch('dulwich.contrib.swift.SwiftConnector', - new_callable=create_swift_connector, - store=store): - swift.SwiftRepo('fakerepo', conf=self.conf) - - def test_init_no_data(self): - with patch('dulwich.contrib.swift.SwiftConnector', - new_callable=create_swift_connector): - self.assertRaises(Exception, swift.SwiftRepo, - 'fakerepo', self.conf) - - def test_init_bad_data(self): - store = {'fakerepo/.git/objects/pack': ''} - with patch('dulwich.contrib.swift.SwiftConnector', - new_callable=create_swift_connector, - store=store): - self.assertRaises(Exception, swift.SwiftRepo, - 'fakerepo', self.conf) - - def test_put_named_file(self): - store = {'fakerepo/objects/pack': ''} - with patch('dulwich.contrib.swift.SwiftConnector', - new_callable=create_swift_connector, - store=store): - repo = swift.SwiftRepo('fakerepo', conf=self.conf) - desc = b'Fake repo' - repo._put_named_file('description', desc) - self.assertEqual(repo.scon.store['fakerepo/description'], - desc) - - def test_init_bare(self): - fsc = FakeSwiftConnector('fakeroot', conf=self.conf) - with patch('dulwich.contrib.swift.SwiftConnector', - new_callable=create_swift_connector, - store=fsc.store): - swift.SwiftRepo.init_bare(fsc, conf=self.conf) - self.assertIn('fakeroot/objects/pack', fsc.store) - self.assertIn('fakeroot/info/refs', fsc.store) - self.assertIn('fakeroot/description', fsc.store) - - -@skipIf(missing_libs, skipmsg) -@skipIfPY3 -class TestPackInfoLoadDump(TestCase): - - def setUp(self): - super(TestPackInfoLoadDump, self).setUp() - conf = swift.load_conf(file=StringIO(config_file % - def_config_file)) - sos = swift.SwiftObjectStore( - FakeSwiftConnector('fakerepo', conf=conf)) - commit_amount = 10 - self.commits = create_commits(length=commit_amount, marker="m") - data = [(d.type_num, d.as_raw_string()) for d in self.commits] - f = BytesIO() - fi = BytesIO() - expected = build_pack(f, data, store=sos) - entries = [(sha, ofs, checksum) for - ofs, _, _, sha, checksum in expected] - self.pack_data = PackData.from_file(file=f, size=None) - write_pack_index_v2( - fi, entries, self.pack_data.calculate_checksum()) - fi.seek(0) - self.pack_index = load_pack_index_file('', fi) - -# def test_pack_info_perf(self): -# dump_time = [] -# load_time = [] -# for i in range(0, 100): -# start = time() -# dumps = swift.pack_info_create(self.pack_data, self.pack_index) -# dump_time.append(time() - start) -# for i in range(0, 100): -# start = time() -# pack_infos = swift.load_pack_info('', file=BytesIO(dumps)) -# load_time.append(time() - start) -# print sum(dump_time) / float(len(dump_time)) -# print sum(load_time) / float(len(load_time)) - - def test_pack_info(self): - dumps = swift.pack_info_create(self.pack_data, self.pack_index) - pack_infos = swift.load_pack_info('', file=BytesIO(dumps)) - for obj in self.commits: - self.assertIn(obj.id, pack_infos) - - -@skipIf(missing_libs, skipmsg) -class TestSwiftInfoRefsContainer(TestCase): - - def setUp(self): - super(TestSwiftInfoRefsContainer, self).setUp() - content = ( - b"22effb216e3a82f97da599b8885a6cadb488b4c5\trefs/heads/master\n" - b"cca703b0e1399008b53a1a236d6b4584737649e4\trefs/heads/dev") - self.store = {'fakerepo/info/refs': content} - self.conf = swift.load_conf(file=StringIO(config_file % - def_config_file)) - self.fsc = FakeSwiftConnector('fakerepo', conf=self.conf) - self.object_store = {} - - def test_init(self): - """info/refs does not exists""" - irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store) - self.assertEqual(len(irc._refs), 0) - self.fsc.store = self.store - irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store) - self.assertIn(b'refs/heads/dev', irc.allkeys()) - self.assertIn(b'refs/heads/master', irc.allkeys()) - - def test_set_if_equals(self): - self.fsc.store = self.store - irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store) - irc.set_if_equals(b'refs/heads/dev', - b"cca703b0e1399008b53a1a236d6b4584737649e4", b'1'*40) - self.assertEqual(irc[b'refs/heads/dev'], b'1'*40) - - def test_remove_if_equals(self): - self.fsc.store = self.store - irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store) - irc.remove_if_equals(b'refs/heads/dev', - b"cca703b0e1399008b53a1a236d6b4584737649e4") - self.assertNotIn(b'refs/heads/dev', irc.allkeys()) - - -@skipIf(missing_libs, skipmsg) -class TestSwiftConnector(TestCase): - - def setUp(self): - super(TestSwiftConnector, self).setUp() - self.conf = swift.load_conf(file=StringIO(config_file % - def_config_file)) - with patch('geventhttpclient.HTTPClient.request', - fake_auth_request_v1): - self.conn = swift.SwiftConnector('fakerepo', conf=self.conf) - - def test_init_connector(self): - self.assertEqual(self.conn.auth_ver, '1') - self.assertEqual(self.conn.auth_url, - 'http://127.0.0.1:8080/auth/v1.0') - self.assertEqual(self.conn.user, 'test:tester') - self.assertEqual(self.conn.password, 'testing') - self.assertEqual(self.conn.root, 'fakerepo') - self.assertEqual(self.conn.storage_url, - 'http://127.0.0.1:8080/v1.0/AUTH_fakeuser') - self.assertEqual(self.conn.token, '12' * 10) - self.assertEqual(self.conn.http_timeout, 1) - self.assertEqual(self.conn.http_pool_length, 1) - self.assertEqual(self.conn.concurrency, 1) - self.conf.set('swift', 'auth_ver', '2') - self.conf.set('swift', 'auth_url', 'http://127.0.0.1:8080/auth/v2.0') - with patch('geventhttpclient.HTTPClient.request', - fake_auth_request_v2): - conn = swift.SwiftConnector('fakerepo', conf=self.conf) - self.assertEqual(conn.user, 'tester') - self.assertEqual(conn.tenant, 'test') - self.conf.set('swift', 'auth_ver', '1') - self.conf.set('swift', 'auth_url', 'http://127.0.0.1:8080/auth/v1.0') - with patch('geventhttpclient.HTTPClient.request', - fake_auth_request_v1_error): - self.assertRaises(swift.SwiftException, - lambda: swift.SwiftConnector('fakerepo', - conf=self.conf)) - - def test_root_exists(self): - with patch('geventhttpclient.HTTPClient.request', - lambda *args: Response()): - self.assertEqual(self.conn.test_root_exists(), True) - - def test_root_not_exists(self): - with patch('geventhttpclient.HTTPClient.request', - lambda *args: Response(status=404)): - self.assertEqual(self.conn.test_root_exists(), None) - - def test_create_root(self): - with patch('dulwich.contrib.swift.SwiftConnector.test_root_exists', - lambda *args: None): - with patch('geventhttpclient.HTTPClient.request', - lambda *args: Response()): - self.assertEqual(self.conn.create_root(), None) - - def test_create_root_fails(self): - with patch('dulwich.contrib.swift.SwiftConnector.test_root_exists', - lambda *args: None): - with patch('geventhttpclient.HTTPClient.request', - lambda *args: Response(status=404)): - self.assertRaises(swift.SwiftException, - lambda: self.conn.create_root()) - - def test_get_container_objects(self): - with patch('geventhttpclient.HTTPClient.request', - lambda *args: Response(content=json_dumps( - (({'name': 'a'}, {'name': 'b'}))))): - self.assertEqual(len(self.conn.get_container_objects()), 2) - - def test_get_container_objects_fails(self): - with patch('geventhttpclient.HTTPClient.request', - lambda *args: Response(status=404)): - self.assertEqual(self.conn.get_container_objects(), None) - - def test_get_object_stat(self): - with patch('geventhttpclient.HTTPClient.request', - lambda *args: Response(headers={'content-length': '10'})): - self.assertEqual(self.conn.get_object_stat('a')['content-length'], - '10') - - def test_get_object_stat_fails(self): - with patch('geventhttpclient.HTTPClient.request', - lambda *args: Response(status=404)): - self.assertEqual(self.conn.get_object_stat('a'), None) - - def test_put_object(self): - with patch('geventhttpclient.HTTPClient.request', - lambda *args, **kwargs: Response()): - self.assertEqual(self.conn.put_object('a', BytesIO(b'content')), - None) - - def test_put_object_fails(self): - with patch('geventhttpclient.HTTPClient.request', - lambda *args, **kwargs: Response(status=400)): - self.assertRaises(swift.SwiftException, - lambda: self.conn.put_object( - 'a', BytesIO(b'content'))) - - def test_get_object(self): - with patch('geventhttpclient.HTTPClient.request', - lambda *args, **kwargs: Response(content=b'content')): - self.assertEqual(self.conn.get_object('a').read(), b'content') - with patch('geventhttpclient.HTTPClient.request', - lambda *args, **kwargs: Response(content=b'content')): - self.assertEqual( - self.conn.get_object('a', range='0-6'), - b'content') - - def test_get_object_fails(self): - with patch('geventhttpclient.HTTPClient.request', - lambda *args, **kwargs: Response(status=404)): - self.assertEqual(self.conn.get_object('a'), None) - - def test_del_object(self): - with patch('geventhttpclient.HTTPClient.request', - lambda *args: Response()): - self.assertEqual(self.conn.del_object('a'), None) - - def test_del_root(self): - with patch('dulwich.contrib.swift.SwiftConnector.del_object', - lambda *args: None): - with patch('dulwich.contrib.swift.SwiftConnector.' - 'get_container_objects', - lambda *args: ({'name': 'a'}, {'name': 'b'})): - with patch('geventhttpclient.HTTPClient.request', - lambda *args: Response()): - self.assertEqual(self.conn.del_root(), None) - - -@skipIf(missing_libs, skipmsg) -class SwiftObjectStoreTests(ObjectStoreTests, TestCase): - - def setUp(self): - TestCase.setUp(self) - conf = swift.load_conf(file=StringIO(config_file % - def_config_file)) - fsc = FakeSwiftConnector('fakerepo', conf=conf) - self.store = swift.SwiftObjectStore(fsc) diff --git a/ext_libs/dulwich/contrib/test_swift_smoke.py b/ext_libs/dulwich/contrib/test_swift_smoke.py deleted file mode 100644 index 2ab960c3..00000000 --- a/ext_libs/dulwich/contrib/test_swift_smoke.py +++ /dev/null @@ -1,317 +0,0 @@ -# test_smoke.py -- Functional tests for the Swift backend. -# Copyright (C) 2013 eNovance SAS -# -# Author: Fabien Boucher -# -# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU -# General Public License as public by the Free Software Foundation; version 2.0 -# or (at your option) any later version. You can redistribute it and/or -# modify it under the terms of either of these two licenses. -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# You should have received a copy of the licenses; if not, see -# for a copy of the GNU General Public License -# and for a copy of the Apache -# License, Version 2.0. -# - -"""Start functional tests - -A Swift installation must be available before -starting those tests. The account and authentication method used -during this functional tests must be changed in the configuration file -passed as environment variable. -The container used to create a fake repository is defined -in cls.fakerepo and will be deleted after the tests. - -DULWICH_SWIFT_CFG=/tmp/conf.cfg PYTHONPATH=. python -m unittest \ - dulwich.tests_swift.test_smoke -""" - -import os -import unittest -import tempfile -import shutil - -import gevent -from gevent import monkey -monkey.patch_all() - -from dulwich import ( # noqa:E402 - server, - repo, - index, - client, - objects, - ) -from dulwich.contrib import swift # noqa:E402 - - -class DulwichServer(): - """Start the TCPGitServer with Swift backend - """ - def __init__(self, backend, port): - self.port = port - self.backend = backend - - def run(self): - self.server = server.TCPGitServer(self.backend, - 'localhost', - port=self.port) - self.job = gevent.spawn(self.server.serve_forever) - - def stop(self): - self.server.shutdown() - gevent.joinall((self.job,)) - - -class SwiftSystemBackend(server.Backend): - - def open_repository(self, path): - return swift.SwiftRepo(path, conf=swift.load_conf()) - - -class SwiftRepoSmokeTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - cls.backend = SwiftSystemBackend() - cls.port = 9148 - cls.server_address = 'localhost' - cls.fakerepo = 'fakerepo' - cls.th_server = DulwichServer(cls.backend, cls.port) - cls.th_server.run() - cls.conf = swift.load_conf() - - @classmethod - def tearDownClass(cls): - cls.th_server.stop() - - def setUp(self): - self.scon = swift.SwiftConnector(self.fakerepo, self.conf) - if self.scon.test_root_exists(): - try: - self.scon.del_root() - except swift.SwiftException: - pass - self.temp_d = tempfile.mkdtemp() - if os.path.isdir(self.temp_d): - shutil.rmtree(self.temp_d) - - def tearDown(self): - if self.scon.test_root_exists(): - try: - self.scon.del_root() - except swift.SwiftException: - pass - if os.path.isdir(self.temp_d): - shutil.rmtree(self.temp_d) - - def test_init_bare(self): - swift.SwiftRepo.init_bare(self.scon, self.conf) - self.assertTrue(self.scon.test_root_exists()) - obj = self.scon.get_container_objects() - filtered = [o for o in obj if o['name'] == 'info/refs' - or o['name'] == 'objects/pack'] - self.assertEqual(len(filtered), 2) - - def test_clone_bare(self): - local_repo = repo.Repo.init(self.temp_d, mkdir=True) - swift.SwiftRepo.init_bare(self.scon, self.conf) - tcp_client = client.TCPGitClient(self.server_address, - port=self.port) - remote_refs = tcp_client.fetch(self.fakerepo, local_repo) - # The remote repo is empty (no refs retreived) - self.assertEqual(remote_refs, None) - - def test_push_commit(self): - def determine_wants(*args): - return {"refs/heads/master": local_repo.refs["HEAD"]} - - local_repo = repo.Repo.init(self.temp_d, mkdir=True) - # Nothing in the staging area - local_repo.do_commit('Test commit', 'fbo@localhost') - sha = local_repo.refs.read_loose_ref('refs/heads/master') - swift.SwiftRepo.init_bare(self.scon, self.conf) - tcp_client = client.TCPGitClient(self.server_address, - port=self.port) - tcp_client.send_pack(self.fakerepo, - determine_wants, - local_repo.object_store.generate_pack_data) - swift_repo = swift.SwiftRepo("fakerepo", self.conf) - remote_sha = swift_repo.refs.read_loose_ref('refs/heads/master') - self.assertEqual(sha, remote_sha) - - def test_push_branch(self): - def determine_wants(*args): - return {"refs/heads/mybranch": - local_repo.refs["refs/heads/mybranch"]} - - local_repo = repo.Repo.init(self.temp_d, mkdir=True) - # Nothing in the staging area - local_repo.do_commit('Test commit', 'fbo@localhost', - ref='refs/heads/mybranch') - sha = local_repo.refs.read_loose_ref('refs/heads/mybranch') - swift.SwiftRepo.init_bare(self.scon, self.conf) - tcp_client = client.TCPGitClient(self.server_address, - port=self.port) - tcp_client.send_pack("/fakerepo", - determine_wants, - local_repo.object_store.generate_pack_data) - swift_repo = swift.SwiftRepo(self.fakerepo, self.conf) - remote_sha = swift_repo.refs.read_loose_ref('refs/heads/mybranch') - self.assertEqual(sha, remote_sha) - - def test_push_multiple_branch(self): - def determine_wants(*args): - return {"refs/heads/mybranch": - local_repo.refs["refs/heads/mybranch"], - "refs/heads/master": - local_repo.refs["refs/heads/master"], - "refs/heads/pullr-108": - local_repo.refs["refs/heads/pullr-108"]} - - local_repo = repo.Repo.init(self.temp_d, mkdir=True) - # Nothing in the staging area - local_shas = {} - remote_shas = {} - for branch in ('master', 'mybranch', 'pullr-108'): - local_shas[branch] = local_repo.do_commit( - 'Test commit %s' % branch, 'fbo@localhost', - ref='refs/heads/%s' % branch) - swift.SwiftRepo.init_bare(self.scon, self.conf) - tcp_client = client.TCPGitClient(self.server_address, - port=self.port) - tcp_client.send_pack(self.fakerepo, - determine_wants, - local_repo.object_store.generate_pack_data) - swift_repo = swift.SwiftRepo("fakerepo", self.conf) - for branch in ('master', 'mybranch', 'pullr-108'): - remote_shas[branch] = swift_repo.refs.read_loose_ref( - 'refs/heads/%s' % branch) - self.assertDictEqual(local_shas, remote_shas) - - def test_push_data_branch(self): - def determine_wants(*args): - return {"refs/heads/master": local_repo.refs["HEAD"]} - local_repo = repo.Repo.init(self.temp_d, mkdir=True) - os.mkdir(os.path.join(self.temp_d, "dir")) - files = ('testfile', 'testfile2', 'dir/testfile3') - i = 0 - for f in files: - open(os.path.join(self.temp_d, f), 'w').write("DATA %s" % i) - i += 1 - local_repo.stage(files) - local_repo.do_commit('Test commit', 'fbo@localhost', - ref='refs/heads/master') - swift.SwiftRepo.init_bare(self.scon, self.conf) - tcp_client = client.TCPGitClient(self.server_address, - port=self.port) - tcp_client.send_pack(self.fakerepo, - determine_wants, - local_repo.object_store.generate_pack_data) - swift_repo = swift.SwiftRepo("fakerepo", self.conf) - commit_sha = swift_repo.refs.read_loose_ref('refs/heads/master') - otype, data = swift_repo.object_store.get_raw(commit_sha) - commit = objects.ShaFile.from_raw_string(otype, data) - otype, data = swift_repo.object_store.get_raw(commit._tree) - tree = objects.ShaFile.from_raw_string(otype, data) - objs = tree.items() - objs_ = [] - for tree_entry in objs: - objs_.append(swift_repo.object_store.get_raw(tree_entry.sha)) - # Blob - self.assertEqual(objs_[1][1], 'DATA 0') - self.assertEqual(objs_[2][1], 'DATA 1') - # Tree - self.assertEqual(objs_[0][0], 2) - - def test_clone_then_push_data(self): - self.test_push_data_branch() - shutil.rmtree(self.temp_d) - local_repo = repo.Repo.init(self.temp_d, mkdir=True) - tcp_client = client.TCPGitClient(self.server_address, - port=self.port) - remote_refs = tcp_client.fetch(self.fakerepo, local_repo) - files = (os.path.join(self.temp_d, 'testfile'), - os.path.join(self.temp_d, 'testfile2')) - local_repo["HEAD"] = remote_refs["refs/heads/master"] - indexfile = local_repo.index_path() - tree = local_repo["HEAD"].tree - index.build_index_from_tree(local_repo.path, indexfile, - local_repo.object_store, tree) - for f in files: - self.assertEqual(os.path.isfile(f), True) - - def determine_wants(*args): - return {"refs/heads/master": local_repo.refs["HEAD"]} - os.mkdir(os.path.join(self.temp_d, "test")) - files = ('testfile11', 'testfile22', 'test/testfile33') - i = 0 - for f in files: - open(os.path.join(self.temp_d, f), 'w').write("DATA %s" % i) - i += 1 - local_repo.stage(files) - local_repo.do_commit('Test commit', 'fbo@localhost', - ref='refs/heads/master') - tcp_client.send_pack("/fakerepo", - determine_wants, - local_repo.object_store.generate_pack_data) - - def test_push_remove_branch(self): - def determine_wants(*args): - return {"refs/heads/pullr-108": objects.ZERO_SHA, - "refs/heads/master": - local_repo.refs['refs/heads/master'], - "refs/heads/mybranch": - local_repo.refs['refs/heads/mybranch'], - } - self.test_push_multiple_branch() - local_repo = repo.Repo(self.temp_d) - tcp_client = client.TCPGitClient(self.server_address, - port=self.port) - tcp_client.send_pack(self.fakerepo, - determine_wants, - local_repo.object_store.generate_pack_data) - swift_repo = swift.SwiftRepo("fakerepo", self.conf) - self.assertNotIn('refs/heads/pullr-108', swift_repo.refs.allkeys()) - - def test_push_annotated_tag(self): - def determine_wants(*args): - return {"refs/heads/master": local_repo.refs["HEAD"], - "refs/tags/v1.0": local_repo.refs["refs/tags/v1.0"]} - local_repo = repo.Repo.init(self.temp_d, mkdir=True) - # Nothing in the staging area - sha = local_repo.do_commit('Test commit', 'fbo@localhost') - otype, data = local_repo.object_store.get_raw(sha) - commit = objects.ShaFile.from_raw_string(otype, data) - tag = objects.Tag() - tag.tagger = "fbo@localhost" - tag.message = "Annotated tag" - tag.tag_timezone = objects.parse_timezone('-0200')[0] - tag.tag_time = commit.author_time - tag.object = (objects.Commit, commit.id) - tag.name = "v0.1" - local_repo.object_store.add_object(tag) - local_repo.refs['refs/tags/v1.0'] = tag.id - swift.SwiftRepo.init_bare(self.scon, self.conf) - tcp_client = client.TCPGitClient(self.server_address, - port=self.port) - tcp_client.send_pack(self.fakerepo, - determine_wants, - local_repo.object_store.generate_pack_data) - swift_repo = swift.SwiftRepo(self.fakerepo, self.conf) - tag_sha = swift_repo.refs.read_loose_ref('refs/tags/v1.0') - otype, data = swift_repo.object_store.get_raw(tag_sha) - rtag = objects.ShaFile.from_raw_string(otype, data) - self.assertEqual(rtag.object[1], commit.id) - self.assertEqual(rtag.id, tag.id) - - -if __name__ == '__main__': - unittest.main() diff --git a/ext_libs/dulwich/diff_tree.py b/ext_libs/dulwich/diff_tree.py index a1e8d63c..a87be78a 100644 --- a/ext_libs/dulwich/diff_tree.py +++ b/ext_libs/dulwich/diff_tree.py @@ -19,7 +19,7 @@ # """Utilities for diffing files and trees.""" -import sys + from collections import ( defaultdict, namedtuple, @@ -314,9 +314,8 @@ def _count_blocks(obj): block_truncate = block.truncate block_getvalue = block.getvalue - for c in chain(*obj.as_raw_chunks()): - if sys.version_info[0] == 3: - c = c.to_bytes(1, 'big') + for c in chain.from_iterable(obj.as_raw_chunks()): + c = c.to_bytes(1, 'big') block_write(c) n += 1 if c == b'\n' or n == _BLOCK_SIZE: @@ -619,6 +618,10 @@ def changes_with_renames(self, tree1_id, tree2_id, want_unchanged=False, _count_blocks_py = _count_blocks try: # Try to import C versions - from dulwich._diff_tree import _is_tree, _merge_entries, _count_blocks + from dulwich._diff_tree import ( # type: ignore + _is_tree, + _merge_entries, + _count_blocks, + ) except ImportError: pass diff --git a/ext_libs/dulwich/errors.py b/ext_libs/dulwich/errors.py index f9821c44..19d5ca73 100644 --- a/ext_libs/dulwich/errors.py +++ b/ext_libs/dulwich/errors.py @@ -117,11 +117,17 @@ class GitProtocolError(Exception): def __init__(self, *args, **kwargs): Exception.__init__(self, *args, **kwargs) + def __eq__(self, other): + return isinstance(self, type(other)) and self.args == other.args + class SendPackError(GitProtocolError): """An error occurred during send_pack.""" +# N.B.: UpdateRefsError is no longer used and will be result in +# Dulwich 0.21. +# remove: >= 0.21 class UpdateRefsError(GitProtocolError): """The server reported errors updating refs.""" @@ -133,9 +139,21 @@ def __init__(self, *args, **kwargs): class HangupException(GitProtocolError): """Hangup exception.""" - def __init__(self): - super(HangupException, self).__init__( - "The remote server unexpectedly closed the connection.") + def __init__(self, stderr_lines=None): + if stderr_lines: + super(HangupException, self).__init__( + '\n'.join( + [line.decode('utf-8', 'surrogateescape') + for line in stderr_lines])) + else: + super(HangupException, self).__init__( + "The remote server unexpectedly closed the connection.") + self.stderr_lines = stderr_lines + + def __eq__(self, other): + return ( + isinstance(self, type(other)) and + self.stderr_lines == other.stderr_lines) class UnexpectedCommandError(GitProtocolError): @@ -162,10 +180,6 @@ class ObjectFormatException(FileFormatException): """Indicates an error parsing an object.""" -class EmptyFileException(FileFormatException): - """An unexpectedly empty file was encountered.""" - - class NoIndexPresent(Exception): """No index is present.""" diff --git a/ext_libs/dulwich/fastexport.py b/ext_libs/dulwich/fastexport.py index 89454164..c4fd1cf6 100644 --- a/ext_libs/dulwich/fastexport.py +++ b/ext_libs/dulwich/fastexport.py @@ -21,8 +21,6 @@ """Fast export/import functionality.""" -import sys - from dulwich.index import ( commit_tree, ) @@ -32,10 +30,6 @@ Tag, ZERO_SHA, ) -from fastimport import __version__ as fastimport_version -if (fastimport_version <= (0, 9, 5) and - sys.version_info[0] == 3 and sys.version_info[1] < 5): - raise ImportError("Older versions of fastimport don't support python3<3.5") from fastimport import ( # noqa: E402 commands, errors as fastimport_errors, diff --git a/ext_libs/dulwich/file.py b/ext_libs/dulwich/file.py index 19d9e2e7..c19a729d 100644 --- a/ext_libs/dulwich/file.py +++ b/ext_libs/dulwich/file.py @@ -20,20 +20,17 @@ """Safe access to git files.""" -import errno import io import os import sys -import tempfile def ensure_dir_exists(dirname): """Ensure a directory exists, creating if necessary.""" try: os.makedirs(dirname) - except OSError as e: - if e.errno != errno.EEXIST: - raise + except FileExistsError: + pass def _fancy_rename(oldname, newname): @@ -45,6 +42,8 @@ def _fancy_rename(oldname, newname): raise return + # Defer the tempfile import since it pulls in a lot of other things. + import tempfile # destination file exists try: (fd, tmpfile) = tempfile.mkstemp(".tmp", prefix=oldname, dir=".") @@ -127,10 +126,8 @@ def __init__(self, filename, mode, bufsize): self._lockfilename, os.O_RDWR | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0)) - except OSError as e: - if e.errno == errno.EEXIST: - raise FileLocked(filename, self._lockfilename) - raise + except FileExistsError: + raise FileLocked(filename, self._lockfilename) self._file = os.fdopen(fd, mode, bufsize) self._closed = False @@ -148,10 +145,8 @@ def abort(self): try: os.remove(self._lockfilename) self._closed = True - except OSError as e: + except FileNotFoundError: # The file may have been removed already, which is ok. - if e.errno != errno.ENOENT: - raise self._closed = True def close(self): diff --git a/ext_libs/dulwich/graph.py b/ext_libs/dulwich/graph.py new file mode 100644 index 00000000..09e21089 --- /dev/null +++ b/ext_libs/dulwich/graph.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab +# Copyright (c) 2020 Kevin B. Hendricks, Stratford Ontario Canada +# +# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU +# General Public License as public by the Free Software Foundation; version 2.0 +# or (at your option) any later version. You can redistribute it and/or +# modify it under the terms of either of these two licenses. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# You should have received a copy of the licenses; if not, see +# for a copy of the GNU General Public License +# and for a copy of the Apache +# License, Version 2.0. + +""" +Implementation of merge-base following the approach of git +""" + +from collections import deque + + +def _find_lcas(lookup_parents, c1, c2s): + cands = [] + cstates = {} + + # Flags to Record State + _ANC_OF_1 = 1 # ancestor of commit 1 + _ANC_OF_2 = 2 # ancestor of commit 2 + _DNC = 4 # Do Not Consider + _LCA = 8 # potential LCA + + def _has_candidates(wlst, cstates): + for cmt in wlst: + if cmt in cstates: + if not (cstates[cmt] & _DNC): + return True + return False + + # initialize the working list + wlst = deque() + cstates[c1] = _ANC_OF_1 + wlst.append(c1) + for c2 in c2s: + cstates[c2] = _ANC_OF_2 + wlst.append(c2) + + # loop until no other LCA candidates are viable in working list + # adding any parents to the list in a breadth first manner + while _has_candidates(wlst, cstates): + cmt = wlst.popleft() + flags = cstates[cmt] + if flags == (_ANC_OF_1 | _ANC_OF_2): + # potential common ancestor + if not (flags & _LCA): + flags = flags | _LCA + cstates[cmt] = flags + cands.append(cmt) + # mark any parents of this node _DNC as all parents + # would be one level further removed common ancestors + flags = flags | _DNC + parents = lookup_parents(cmt) + if parents: + for pcmt in parents: + if pcmt in cstates: + cstates[pcmt] = cstates[pcmt] | flags + else: + cstates[pcmt] = flags + wlst.append(pcmt) + + # walk final candidates removing any superceded by _DNC by later lower LCAs + results = [] + for cmt in cands: + if not (cstates[cmt] & _DNC): + results.append(cmt) + return results + + +def find_merge_base(repo, commit_ids): + """Find lowest common ancestors of commit_ids[0] and *any* of commits_ids[1:] + + Args: + repo: Repository object + commit_ids: list of commit ids + Returns: + list of lowest common ancestor commit_ids + """ + if not commit_ids: + return [] + c1 = commit_ids[0] + if not len(commit_ids) > 1: + return [c1] + c2s = commit_ids[1:] + if c1 in c2s: + return [c1] + parents_provider = repo.parents_provider() + return _find_lcas(parents_provider.get_parents, c1, c2s) + + +def find_octopus_base(repo, commit_ids): + """Find lowest common ancestors of *all* provided commit_ids + + Args: + repo: Repository + commit_ids: list of commit ids + Returns: + list of lowest common ancestor commit_ids + """ + + if not commit_ids: + return [] + if len(commit_ids) <= 2: + return find_merge_base(repo, commit_ids) + parents_provider = repo.parents_provider() + lcas = [commit_ids[0]] + others = commit_ids[1:] + for cmt in others: + next_lcas = [] + for ca in lcas: + res = _find_lcas(parents_provider.get_parents, cmt, [ca]) + next_lcas.extend(res) + lcas = next_lcas[:] + return lcas + + +def can_fast_forward(repo, c1, c2): + """Is it possible to fast-forward from c1 to c2? + + Args: + repo: Repository to retrieve objects from + c1: Commit id for first commit + c2: Commit id for second commit + """ + if c1 == c2: + return True + + # Algorithm: Find the common ancestor + parents_provider = repo.parents_provider() + lcas = _find_lcas(parents_provider.get_parents, c1, [c2]) + return lcas == [c1] diff --git a/ext_libs/dulwich/hooks.py b/ext_libs/dulwich/hooks.py index def9c600..fef044dc 100644 --- a/ext_libs/dulwich/hooks.py +++ b/ext_libs/dulwich/hooks.py @@ -22,8 +22,6 @@ import os import subprocess -import sys -import tempfile from dulwich.errors import ( HookError, @@ -82,11 +80,6 @@ def __init__(self, name, path, numparam, self.cwd = cwd - if sys.version_info[0] == 2 and sys.platform == 'win32': - # Python 2 on windows does not support unicode file paths - # http://bugs.python.org/issue1759845 - self.filepath = self.filepath.encode(sys.getfilesystemencoding()) - def execute(self, *args): """Execute the hook with given args""" @@ -103,8 +96,8 @@ def execute(self, *args): if ret != 0: if (self.post_exec_callback is not None): self.post_exec_callback(0, *args) - raise HookError("Hook %s exited with non-zero status" - % (self.name)) + raise HookError("Hook %s exited with non-zero status %d" + % (self.name, ret)) if (self.post_exec_callback is not None): return self.post_exec_callback(1, *args) except OSError: # no file. silent failure. @@ -143,6 +136,7 @@ def __init__(self, controldir): filepath = os.path.join(controldir, 'hooks', 'commit-msg') def prepare_msg(*args): + import tempfile (fd, path) = tempfile.mkstemp() with os.fdopen(fd, 'wb') as f: @@ -160,3 +154,43 @@ def clean_msg(success, *args): ShellHook.__init__(self, 'commit-msg', filepath, 1, prepare_msg, clean_msg, controldir) + + +class PostReceiveShellHook(ShellHook): + """post-receive shell hook""" + + def __init__(self, controldir): + self.controldir = controldir + filepath = os.path.join(controldir, 'hooks', 'post-receive') + ShellHook.__init__(self, 'post-receive', filepath, 0) + + def execute(self, client_refs): + # do nothing if the script doesn't exist + if not os.path.exists(self.filepath): + return None + + try: + env = os.environ.copy() + env['GIT_DIR'] = self.controldir + + p = subprocess.Popen( + self.filepath, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env + ) + + # client_refs is a list of (oldsha, newsha, ref) + in_data = '\n'.join([' '.join(ref) for ref in client_refs]) + + out_data, err_data = p.communicate(in_data) + + if (p.returncode != 0) or err_data: + err_fmt = "post-receive exit code: %d\n" \ + + "stdout:\n%s\nstderr:\n%s" + err_msg = err_fmt % (p.returncode, out_data, err_data) + raise HookError(err_msg) + return out_data + except OSError as err: + raise HookError(repr(err)) diff --git a/ext_libs/dulwich/ignore.py b/ext_libs/dulwich/ignore.py index 37a0d9ec..fb81c7fc 100644 --- a/ext_libs/dulwich/ignore.py +++ b/ext_libs/dulwich/ignore.py @@ -24,10 +24,23 @@ import os.path import re -import sys +from typing import ( + BinaryIO, + Iterable, + List, + Optional, + TYPE_CHECKING, + Dict, + Union, + ) +if TYPE_CHECKING: + from dulwich.repo import Repo -def _translate_segment(segment): +from dulwich.config import get_xdg_config_home_path, Config + + +def _translate_segment(segment: bytes) -> bytes: if segment == b"*": return b'[^/]+' res = b"" @@ -62,7 +75,7 @@ def _translate_segment(segment): return res -def translate(pat): +def translate(pat: bytes) -> bytes: """Translate a shell PATTERN to a regular expression. There is no way to quote meta-characters. @@ -99,7 +112,7 @@ def translate(pat): return res + b'\\Z' -def read_ignore_patterns(f): +def read_ignore_patterns(f: BinaryIO) -> Iterable[bytes]: """Read a git ignore file. Args: @@ -126,7 +139,8 @@ def read_ignore_patterns(f): yield line -def match_pattern(path, pattern, ignorecase=False): +def match_pattern( + path: bytes, pattern: bytes, ignorecase: bool = False) -> bool: """Match a gitignore-style pattern against a path. Args: @@ -142,7 +156,7 @@ def match_pattern(path, pattern, ignorecase=False): class Pattern(object): """A single ignore pattern.""" - def __init__(self, pattern, ignorecase=False): + def __init__(self, pattern: bytes, ignorecase: bool = False): self.pattern = pattern self.ignorecase = ignorecase if pattern[0:1] == b'!': @@ -157,22 +171,22 @@ def __init__(self, pattern, ignorecase=False): flags = re.IGNORECASE self._re = re.compile(translate(pattern), flags) - def __bytes__(self): + def __bytes__(self) -> bytes: return self.pattern - def __str__(self): - return self.pattern.decode(sys.getfilesystemencoding()) + def __str__(self) -> str: + return os.fsdecode(self.pattern) - def __eq__(self, other): - return (type(self) == type(other) and + def __eq__(self, other: object) -> bool: + return (isinstance(other, type(self)) and self.pattern == other.pattern and self.ignorecase == other.ignorecase) - def __repr__(self): - return "%s(%s, %r)" % ( + def __repr__(self) -> str: + return "%s(%r, %r)" % ( type(self).__name__, self.pattern, self.ignorecase) - def match(self, path): + def match(self, path: bytes) -> bool: """Try to match a path against this ignore pattern. Args: @@ -184,31 +198,33 @@ def match(self, path): class IgnoreFilter(object): - def __init__(self, patterns, ignorecase=False): - self._patterns = [] + def __init__(self, patterns: Iterable[bytes], ignorecase: bool = False, + path=None): + self._patterns = [] # type: List[Pattern] self._ignorecase = ignorecase + self._path = path for pattern in patterns: self.append_pattern(pattern) - def append_pattern(self, pattern): + def append_pattern(self, pattern: bytes) -> None: """Add a pattern to the set.""" self._patterns.append(Pattern(pattern, self._ignorecase)) - def find_matching(self, path): + def find_matching(self, path: Union[bytes, str]) -> Iterable[Pattern]: """Yield all matching patterns for path. Args: path: Path to match Returns: - Iterator over iterators + Iterator over iterators """ if not isinstance(path, bytes): - path = path.encode(sys.getfilesystemencoding()) + path = os.fsencode(path) for pattern in self._patterns: if pattern.match(path): yield pattern - def is_ignored(self, path): + def is_ignored(self, path: bytes) -> Optional[bool]: """Check whether a path is ignored. For directories, include a trailing slash. @@ -222,17 +238,17 @@ def is_ignored(self, path): return status @classmethod - def from_path(cls, path, ignorecase=False): + def from_path(cls, path, ignorecase: bool = False) -> 'IgnoreFilter': with open(path, 'rb') as f: - ret = cls(read_ignore_patterns(f), ignorecase) - ret._path = path - return ret + return cls(read_ignore_patterns(f), ignorecase, path=path) - def __repr__(self): - if getattr(self, '_path', None) is None: - return "<%s>" % (type(self).__name__) + def __repr__(self) -> str: + path = getattr(self, '_path', None) + if path is not None: + return "%s.from_path(%r)" % ( + type(self).__name__, path) else: - return "%s.from_path(%r)" % (type(self).__name__, self._path) + return "<%s>" % (type(self).__name__) class IgnoreFilterStack(object): @@ -241,7 +257,7 @@ class IgnoreFilterStack(object): def __init__(self, filters): self._filters = filters - def is_ignored(self, path): + def is_ignored(self, path: str) -> Optional[bool]: """Check whether a path is explicitly included or excluded in ignores. Args: @@ -258,7 +274,7 @@ def is_ignored(self, path): return status -def default_user_ignore_filter_path(config): +def default_user_ignore_filter_path(config: Config) -> str: """Return default user ignore filter path. Args: @@ -271,26 +287,27 @@ def default_user_ignore_filter_path(config): except KeyError: pass - xdg_config_home = os.environ.get("XDG_CONFIG_HOME", "~/.config/") - return os.path.join(xdg_config_home, 'git', 'ignore') + return get_xdg_config_home_path('git', 'ignore') class IgnoreFilterManager(object): """Ignore file manager.""" - def __init__(self, top_path, global_filters, ignorecase): - self._path_filters = {} + def __init__( + self, top_path: str, global_filters: List[IgnoreFilter], + ignorecase: bool): + self._path_filters = {} # type: Dict[str, Optional[IgnoreFilter]] self._top_path = top_path self._global_filters = global_filters self._ignorecase = ignorecase - def __repr__(self): + def __repr__(self) -> str: return "%s(%s, %r, %r)" % ( type(self).__name__, self._top_path, self._global_filters, self._ignorecase) - def _load_path(self, path): + def _load_path(self, path: str) -> Optional[IgnoreFilter]: try: return self._path_filters[path] except KeyError: @@ -304,7 +321,7 @@ def _load_path(self, path): self._path_filters[path] = None return self._path_filters[path] - def find_matching(self, path): + def find_matching(self, path: str) -> Iterable[Pattern]: """Find matching patterns for path. Stops after the first ignore file with matches. @@ -336,7 +353,7 @@ def find_matching(self, path): filters.insert(0, (i, ignore_filter)) return iter([]) - def is_ignored(self, path): + def is_ignored(self, path: str) -> Optional[bool]: """Check whether a path is explicitly included or excluded in ignores. Args: @@ -351,7 +368,7 @@ def is_ignored(self, path): return None @classmethod - def from_repo(cls, repo): + def from_repo(cls, repo: 'Repo') -> 'IgnoreFilterManager': """Create a IgnoreFilterManager from a repository. Args: diff --git a/ext_libs/dulwich/index.py b/ext_libs/dulwich/index.py index 3101a27f..88e2c72e 100644 --- a/ext_libs/dulwich/index.py +++ b/ext_libs/dulwich/index.py @@ -21,11 +21,25 @@ """Parser for the git index file format.""" import collections -import errno import os import stat import struct import sys +from typing import ( + Any, + BinaryIO, + Callable, + Dict, + List, + Optional, + TYPE_CHECKING, + Iterable, + Iterator, + Tuple, + ) + +if TYPE_CHECKING: + from dulwich.object_store import BaseObjectStore from dulwich.file import GitFile from dulwich.objects import ( @@ -53,6 +67,9 @@ FLAG_EXTENDED = 0x4000 +DEFAULT_VERSION = 2 + + def pathsplit(path): """Split a /-delimited path into a directory part and a basename. @@ -146,7 +163,7 @@ def write_cache_entry(f, entry): f.write(b'\0' * ((beginoffset + real_size) - f.tell())) -def read_index(f): +def read_index(f: BinaryIO): """Read an index file, yielding the individual entries.""" header = f.read(4) if header != b'DIRC': @@ -169,36 +186,45 @@ def read_index_dict(f): return ret -def write_index(f, entries): +def write_index( + f: BinaryIO, + entries: List[Any], version: Optional[int] = None): """Write an index file. Args: f: File-like object to write to + version: Version number to write entries: Iterable over the entries to write """ + if version is None: + version = DEFAULT_VERSION f.write(b'DIRC') - f.write(struct.pack(b'>LL', 2, len(entries))) + f.write(struct.pack(b'>LL', version, len(entries))) for x in entries: write_cache_entry(f, x) -def write_index_dict(f, entries): +def write_index_dict( + f: BinaryIO, entries: Dict[bytes, IndexEntry], + version: Optional[int] = None) -> None: """Write an index file based on the contents of a dictionary. """ entries_list = [] for name in sorted(entries): entries_list.append((name,) + tuple(entries[name])) - write_index(f, entries_list) + write_index(f, entries_list, version=version) -def cleanup_mode(mode): +def cleanup_mode(mode: int) -> int: """Cleanup a mode value. This will return a mode that can be stored in a tree object. Args: mode: Mode to clean up. + Returns: + mode """ if stat.S_ISLNK(mode): return stat.S_IFLNK @@ -222,6 +248,8 @@ def __init__(self, filename): filename: Path to the index file """ self._filename = filename + # TODO(jelmer): Store the version returned by read_index + self._version = None self.clear() self.read() @@ -232,12 +260,12 @@ def path(self): def __repr__(self): return "%s(%r)" % (self.__class__.__name__, self._filename) - def write(self): + def write(self) -> None: """Write current contents of index to disk.""" f = GitFile(self._filename, 'wb') try: f = SHA1Writer(f) - write_index_dict(f, self._byname) + write_index_dict(f, self._byname, version=self._version) finally: f.close() @@ -256,11 +284,11 @@ def read(self): finally: f.close() - def __len__(self): + def __len__(self) -> int: """Number of entries in this index file.""" return len(self._byname) - def __getitem__(self, name): + def __getitem__(self, name: bytes) -> IndexEntry: """Retrieve entry by relative path. Returns: tuple with (ctime, mtime, dev, ino, mode, uid, gid, size, sha, @@ -268,19 +296,19 @@ def __getitem__(self, name): """ return self._byname[name] - def __iter__(self): + def __iter__(self) -> Iterator[bytes]: """Iterate over the paths in this index.""" return iter(self._byname) - def get_sha1(self, path): + def get_sha1(self, path: bytes) -> bytes: """Return the (git object) SHA1 for the object at a path.""" return self[path].sha - def get_mode(self, path): + def get_mode(self, path: bytes) -> int: """Return the POSIX file mode for the object at a path.""" return self[path].mode - def iterobjects(self): + def iterobjects(self) -> Iterable[Tuple[bytes, bytes, int]]: """Iterate over path, sha, mode tuples for use with commit_tree.""" for path in self: entry = self[path] @@ -344,7 +372,9 @@ def commit(self, object_store): return commit_tree(object_store, self.iterobjects()) -def commit_tree(object_store, blobs): +def commit_tree( + object_store: 'BaseObjectStore', + blobs: Iterable[Tuple[bytes, bytes, int]]) -> bytes: """Commit a new tree. Args: @@ -353,8 +383,7 @@ def commit_tree(object_store, blobs): Returns: SHA1 of the created tree. """ - - trees = {b'': {}} + trees = {b'': {}} # type: Dict[bytes, Any] def add_tree(path): if path in trees: @@ -386,7 +415,7 @@ def build_tree(path): return build_tree(b'') -def commit_index(object_store, index): +def commit_index(object_store: 'BaseObjectStore', index: Index) -> bytes: """Create a new tree from an index. Args: @@ -398,8 +427,15 @@ def commit_index(object_store, index): return commit_tree(object_store, index.iterobjects()) -def changes_from_tree(names, lookup_entry, object_store, tree, - want_unchanged=False): +def changes_from_tree( + names: Iterable[bytes], + lookup_entry: Callable[[bytes], Tuple[bytes, int]], + object_store: 'BaseObjectStore', tree: Optional[bytes], + want_unchanged=False) -> Iterable[ + Tuple[ + Tuple[Optional[bytes], Optional[bytes]], + Tuple[Optional[int], Optional[int]], + Tuple[Optional[bytes], Optional[bytes]]]]: """Find the differences between the contents of a tree and a working copy. @@ -437,7 +473,9 @@ def changes_from_tree(names, lookup_entry, object_store, tree, yield ((None, name), (None, other_mode), (None, other_sha)) -def index_entry_from_stat(stat_val, hex_sha, flags, mode=None): +def index_entry_from_stat( + stat_val, hex_sha: bytes, flags: int, + mode: Optional[int] = None): """Create a new index entry from a stat value. Args: @@ -454,7 +492,8 @@ def index_entry_from_stat(stat_val, hex_sha, flags, mode=None): stat_val.st_gid, stat_val.st_size, hex_sha, flags) -def build_file_from_blob(blob, mode, target_path, honor_filemode=True): +def build_file_from_blob(blob, mode, target_path, honor_filemode=True, + tree_encoding='utf-8'): """Build a file or symlink on disk based on a Git object. Args: @@ -467,20 +506,15 @@ def build_file_from_blob(blob, mode, target_path, honor_filemode=True): """ try: oldstat = os.lstat(target_path) - except OSError as e: - if e.errno == errno.ENOENT: - oldstat = None - else: - raise + except FileNotFoundError: + oldstat = None contents = blob.as_raw_string() if stat.S_ISLNK(mode): # FIXME: This will fail on Windows. What should we do instead? if oldstat: os.unlink(target_path) - if sys.platform == 'win32' and sys.version_info[0] == 3: + if sys.platform == 'win32': # os.readlink on Python3 on Windows requires a unicode string. - # TODO(jelmer): Don't assume tree_encoding == fs_encoding - tree_encoding = sys.getfilesystemencoding() contents = contents.decode(tree_encoding) target_path = target_path.decode(tree_encoding) os.symlink(contents, target_path) @@ -547,7 +581,7 @@ def build_index_from_tree(root_path, index_path, object_store, tree_id, index = Index(index_path) if not isinstance(root_path, bytes): - root_path = root_path.encode(sys.getfilesystemencoding()) + root_path = os.fsencode(root_path) for entry in object_store.iter_tree_contents(tree_id): if not validate_path(entry.path, validate_path_element): @@ -567,6 +601,7 @@ def build_index_from_tree(root_path, index_path, object_store, tree_id, obj = object_store[entry.sha] st = build_file_from_blob( obj, entry.mode, full_path, honor_filemode=honor_filemode) + # Add file to index if not honor_filemode or S_ISGITLINK(entry.mode): # we can not use tuple slicing to build a new tuple, @@ -581,7 +616,7 @@ def build_index_from_tree(root_path, index_path, object_store, tree_id, index.write() -def blob_from_path_and_stat(fs_path, st): +def blob_from_path_and_mode(fs_path, mode, tree_encoding='utf-8'): """Create a blob from a path and a stat object. Args: @@ -591,21 +626,30 @@ def blob_from_path_and_stat(fs_path, st): """ assert isinstance(fs_path, bytes) blob = Blob() - if not stat.S_ISLNK(st.st_mode): - with open(fs_path, 'rb') as f: - blob.data = f.read() - else: - if sys.platform == 'win32' and sys.version_info[0] == 3: + if stat.S_ISLNK(mode): + if sys.platform == 'win32': # os.readlink on Python3 on Windows requires a unicode string. - # TODO(jelmer): Don't assume tree_encoding == fs_encoding - tree_encoding = sys.getfilesystemencoding() - fs_path = fs_path.decode(tree_encoding) + fs_path = os.fsdecode(fs_path) blob.data = os.readlink(fs_path).encode(tree_encoding) else: blob.data = os.readlink(fs_path) + else: + with open(fs_path, 'rb') as f: + blob.data = f.read() return blob +def blob_from_path_and_stat(fs_path, st, tree_encoding='utf-8'): + """Create a blob from a path and a stat object. + + Args: + fs_path: Full file system path to file + st: A stat object + Returns: A `Blob` object + """ + return blob_from_path_and_mode(fs_path, st.st_mode, tree_encoding) + + def read_submodule_head(path): """Read the head commit of a submodule. @@ -618,7 +662,7 @@ def read_submodule_head(path): # Repo currently expects a "str", so decode if necessary. # TODO(jelmer): Perhaps move this into Repo() ? if not isinstance(path, str): - path = path.decode(sys.getfilesystemencoding()) + path = os.fsdecode(path) try: repo = Repo(path) except NotGitRepository: @@ -654,7 +698,7 @@ def _has_directory_changed(tree_path, entry): return False -def get_unstaged_changes(index, root_path, filter_blob_callback=None): +def get_unstaged_changes(index: Index, root_path, filter_blob_callback=None): """Walk through an index and check for differences against working tree. Args: @@ -664,7 +708,7 @@ def get_unstaged_changes(index, root_path, filter_blob_callback=None): """ # For each entry in the index check the sha1 & ensure not staged if not isinstance(root_path, bytes): - root_path = root_path.encode(sys.getfilesystemencoding()) + root_path = os.fsencode(root_path) for tree_path, entry in index.iteritems(): full_path = _tree_to_fs_path(root_path, tree_path) @@ -675,17 +719,17 @@ def get_unstaged_changes(index, root_path, filter_blob_callback=None): yield tree_path continue + if not stat.S_ISREG(st.st_mode) and not stat.S_ISLNK(st.st_mode): + continue + blob = blob_from_path_and_stat(full_path, st) if filter_blob_callback is not None: blob = filter_blob_callback(blob, tree_path) - except EnvironmentError as e: - if e.errno == errno.ENOENT: - # The file was removed, so we assume that counts as - # different from whatever file used to exist. - yield tree_path - else: - raise + except FileNotFoundError: + # The file was removed, so we assume that counts as + # different from whatever file used to exist. + yield tree_path else: if blob.id != entry.sha: yield tree_path @@ -694,7 +738,7 @@ def get_unstaged_changes(index, root_path, filter_blob_callback=None): os_sep_bytes = os.sep.encode('ascii') -def _tree_to_fs_path(root_path, tree_path): +def _tree_to_fs_path(root_path, tree_path: bytes): """Convert a git tree path to a file system path. Args: @@ -711,19 +755,16 @@ def _tree_to_fs_path(root_path, tree_path): return os.path.join(root_path, sep_corrected_path) -def _fs_to_tree_path(fs_path, fs_encoding=None): +def _fs_to_tree_path(fs_path): """Convert a file system path to a git tree path. Args: fs_path: File system path. - fs_encoding: File system encoding Returns: Git tree path as bytes """ - if fs_encoding is None: - fs_encoding = sys.getfilesystemencoding() if not isinstance(fs_path, bytes): - fs_path_bytes = fs_path.encode(fs_encoding) + fs_path_bytes = os.fsencode(fs_path) else: fs_path_bytes = fs_path if os_sep_bytes != b'/': @@ -757,13 +798,17 @@ def index_entry_from_path(path, object_store=None): st, head, 0, mode=S_IFGITLINK) return None - blob = blob_from_path_and_stat(path, st) - if object_store is not None: - object_store.add_object(blob) - return index_entry_from_stat(st, blob.id, 0) + if stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode): + blob = blob_from_path_and_stat(path, st) + if object_store is not None: + object_store.add_object(blob) + return index_entry_from_stat(st, blob.id, 0) + return None -def iter_fresh_entries(paths, root_path, object_store=None): + +def iter_fresh_entries( + paths, root_path, object_store: Optional['BaseObjectStore'] = None): """Iterate over current versions of index entries on disk. Args: @@ -776,11 +821,8 @@ def iter_fresh_entries(paths, root_path, object_store=None): p = _tree_to_fs_path(root_path, path) try: entry = index_entry_from_path(p, object_store=object_store) - except EnvironmentError as e: - if e.errno in (errno.ENOENT, errno.EISDIR): - entry = None - else: - raise + except (FileNotFoundError, IsADirectoryError): + entry = None yield path, entry @@ -812,7 +854,6 @@ def iter_fresh_objects(paths, root_path, include_deleted=False, """Iterate over versions of objecs on disk referenced by index. Args: - index: Index file root_path: Root path to access from include_deleted: Include deleted entries with sha and mode set to None diff --git a/ext_libs/dulwich/lfs.py b/ext_libs/dulwich/lfs.py new file mode 100644 index 00000000..8d14dfea --- /dev/null +++ b/ext_libs/dulwich/lfs.py @@ -0,0 +1,75 @@ +# lfs.py -- Implementation of the LFS +# Copyright (C) 2020 Jelmer Vernooij +# +# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU +# General Public License as public by the Free Software Foundation; version 2.0 +# or (at your option) any later version. You can redistribute it and/or +# modify it under the terms of either of these two licenses. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# You should have received a copy of the licenses; if not, see +# for a copy of the GNU General Public License +# and for a copy of the Apache +# License, Version 2.0. +# + +import hashlib +import os +import tempfile + + +class LFSStore(object): + """Stores objects on disk, indexed by SHA256.""" + + def __init__(self, path): + self.path = path + + @classmethod + def create(cls, lfs_dir): + if not os.path.isdir(lfs_dir): + os.mkdir(lfs_dir) + os.mkdir(os.path.join(lfs_dir, 'tmp')) + os.mkdir(os.path.join(lfs_dir, 'objects')) + return cls(lfs_dir) + + @classmethod + def from_repo(cls, repo, create=False): + lfs_dir = os.path.join(repo.controldir, 'lfs') + if create: + return cls.create(lfs_dir) + return cls(lfs_dir) + + def _sha_path(self, sha): + return os.path.join(self.path, 'objects', sha[0:2], sha[2:4], sha) + + def open_object(self, sha): + """Open an object by sha.""" + try: + return open(self._sha_path(sha), 'rb') + except FileNotFoundError: + raise KeyError(sha) + + def write_object(self, chunks): + """Write an object. + + Returns: object SHA + """ + sha = hashlib.sha256() + tmpdir = os.path.join(self.path, 'tmp') + with tempfile.NamedTemporaryFile( + dir=tmpdir, mode='wb', delete=False) as f: + for chunk in chunks: + sha.update(chunk) + f.write(chunk) + f.flush() + tmppath = f.name + path = self._sha_path(sha.hexdigest()) + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + os.rename(tmppath, path) + return sha.hexdigest() diff --git a/ext_libs/dulwich/object_store.py b/ext_libs/dulwich/object_store.py index 5cfea792..32878bb1 100644 --- a/ext_libs/dulwich/object_store.py +++ b/ext_libs/dulwich/object_store.py @@ -23,11 +23,9 @@ """Git object store interfaces and implementation.""" from io import BytesIO -import errno import os import stat import sys -import tempfile from dulwich.diff_tree import ( tree_changes, @@ -48,6 +46,7 @@ hex_to_filename, S_ISGITLINK, object_class, + valid_hexsha, ) from dulwich.pack import ( Pack, @@ -152,7 +151,9 @@ def add_pack_data(self, count, pack_data, progress=None): return f, commit, abort = self.add_pack() try: - write_pack_data(f, count, pack_data, progress) + write_pack_data( + f, count, pack_data, progress, + compression_level=self.pack_compression_level) except BaseException: abort() raise @@ -199,7 +200,7 @@ def iter_tree_contents(self, tree_id, include_trees=False): not stat.S_ISDIR(entry.mode)) or include_trees): yield entry - def find_missing_objects(self, haves, wants, progress=None, + def find_missing_objects(self, haves, wants, shallow=None, progress=None, get_tagged=None, get_parents=lambda commit: commit.parents, depth=None): @@ -208,6 +209,7 @@ def find_missing_objects(self, haves, wants, progress=None, Args: haves: Iterable over SHAs already in common. wants: Iterable over SHAs of objects to fetch. + shallow: Set of shallow commit SHA1s to skip progress: Simple progress function that will be called with updated progress strings. get_tagged: Function that returns a dict of pointed-to sha -> @@ -216,8 +218,8 @@ def find_missing_objects(self, haves, wants, progress=None, commit. Returns: Iterator over (sha, path) pairs. """ - finder = MissingObjectFinder(self, haves, wants, progress, get_tagged, - get_parents=get_parents) + finder = MissingObjectFinder(self, haves, wants, shallow, progress, + get_tagged, get_parents=get_parents) return iter(finder.next, None) def find_common_revisions(self, graphwalker): @@ -236,28 +238,32 @@ def find_common_revisions(self, graphwalker): sha = next(graphwalker) return haves - def generate_pack_contents(self, have, want, progress=None): + def generate_pack_contents(self, have, want, shallow=None, progress=None): """Iterate over the contents of a pack file. Args: have: List of SHA1s of objects that should not be sent want: List of SHA1s of objects that should be sent + shallow: Set of shallow commit SHA1s to skip progress: Optional progress reporting method """ - return self.iter_shas(self.find_missing_objects(have, want, progress)) + missing = self.find_missing_objects(have, want, shallow, progress) + return self.iter_shas(missing) - def generate_pack_data(self, have, want, progress=None, ofs_delta=True): + def generate_pack_data(self, have, want, shallow=None, progress=None, + ofs_delta=True): """Generate pack data objects for a set of wants/haves. Args: have: List of SHA1s of objects that should not be sent want: List of SHA1s of objects that should be sent + shallow: Set of shallow commit SHA1s to skip ofs_delta: Whether OFS deltas can be included progress: Optional progress reporting method """ # TODO(jelmer): More efficient implementation return pack_objects_to_data( - self.generate_pack_contents(have, want, progress)) + self.generate_pack_contents(have, want, shallow, progress)) def peel_sha(self, sha): """Peel all tags from a SHA. @@ -275,7 +281,7 @@ def peel_sha(self, sha): obj = self[sha] return obj - def _collect_ancestors(self, heads, common=set(), + def _collect_ancestors(self, heads, common=set(), shallow=set(), get_parents=lambda commit: commit.parents): """Collect all ancestors of heads up to (excluding) those in common. @@ -299,6 +305,8 @@ def _collect_ancestors(self, heads, common=set(), bases.add(e) elif e not in commits: commits.add(e) + if e in shallow: + continue cmt = self[e] queue.extend(get_parents(cmt)) return (commits, bases) @@ -310,8 +318,9 @@ def close(self): class PackBasedObjectStore(BaseObjectStore): - def __init__(self): + def __init__(self, pack_compression_level=-1): self._pack_cache = {} + self.pack_compression_level = pack_compression_level @property def alternates(self): @@ -512,20 +521,45 @@ def add_objects(self, objects, progress=None): class DiskObjectStore(PackBasedObjectStore): """Git-style object store that exists on disk.""" - def __init__(self, path): + def __init__(self, path, loose_compression_level=-1, + pack_compression_level=-1): """Open an object store. Args: path: Path of the object store. + loose_compression_level: zlib compression level for loose objects + pack_compression_level: zlib compression level for pack objects """ - super(DiskObjectStore, self).__init__() + super(DiskObjectStore, self).__init__( + pack_compression_level=pack_compression_level) self.path = path self.pack_dir = os.path.join(self.path, PACKDIR) self._alternates = None + self.loose_compression_level = loose_compression_level + self.pack_compression_level = pack_compression_level def __repr__(self): return "<%s(%r)>" % (self.__class__.__name__, self.path) + @classmethod + def from_config(cls, path, config): + try: + default_compression_level = int(config.get( + (b'core', ), b'compression').decode()) + except KeyError: + default_compression_level = -1 + try: + loose_compression_level = int(config.get( + (b'core', ), b'looseCompression').decode()) + except KeyError: + loose_compression_level = default_compression_level + try: + pack_compression_level = int(config.get( + (b'core', ), 'packCompression').decode()) + except KeyError: + pack_compression_level = default_compression_level + return cls(path, loose_compression_level, pack_compression_level) + @property def alternates(self): if self._alternates is not None: @@ -538,40 +572,35 @@ def alternates(self): def _read_alternate_paths(self): try: f = GitFile(os.path.join(self.path, INFODIR, "alternates"), 'rb') - except (OSError, IOError) as e: - if e.errno == errno.ENOENT: - return - raise + except FileNotFoundError: + return with f: for line in f.readlines(): line = line.rstrip(b"\n") if line[0] == b"#": continue if os.path.isabs(line): - yield line.decode(sys.getfilesystemencoding()) + yield os.fsdecode(line) else: - yield os.path.join(self.path, line).decode( - sys.getfilesystemencoding()) + yield os.fsdecode(os.path.join(self.path, line)) def add_alternate_path(self, path): """Add an alternate path to this object store. """ try: os.mkdir(os.path.join(self.path, INFODIR)) - except OSError as e: - if e.errno != errno.EEXIST: - raise + except FileExistsError: + pass alternates_path = os.path.join(self.path, INFODIR, "alternates") with GitFile(alternates_path, 'wb') as f: try: orig_f = open(alternates_path, 'rb') - except (OSError, IOError) as e: - if e.errno != errno.ENOENT: - raise + except FileNotFoundError: + pass else: with orig_f: f.write(orig_f.read()) - f.write(path.encode(sys.getfilesystemencoding()) + b"\n") + f.write(os.fsencode(path) + b"\n") if not os.path.isabs(path): path = os.path.join(self.path, path) @@ -581,11 +610,9 @@ def _update_pack_cache(self): """Read and iterate over new pack files and cache them.""" try: pack_dir_contents = os.listdir(self.pack_dir) - except OSError as e: - if e.errno == errno.ENOENT: - self.close() - return [] - raise + except FileNotFoundError: + self.close() + return [] pack_files = set() for name in pack_dir_contents: if name.startswith("pack-") and name.endswith(".pack"): @@ -617,16 +644,17 @@ def _iter_loose_objects(self): if len(base) != 2: continue for rest in os.listdir(os.path.join(self.path, base)): - yield (base+rest).encode(sys.getfilesystemencoding()) + sha = os.fsencode(base+rest) + if not valid_hexsha(sha): + continue + yield sha def _get_loose_object(self, sha): path = self._get_shafile_path(sha) try: return ShaFile.from_path(path) - except (OSError, IOError) as e: - if e.errno == errno.ENOENT: - return None - raise + except FileNotFoundError: + return None def _remove_loose_object(self, sha): os.remove(self._get_shafile_path(sha)) @@ -678,7 +706,9 @@ def _complete_thin_pack(self, f, path, copier, indexer): assert len(ext_sha) == 20 type_num, data = self.get_raw(ext_sha) offset = f.tell() - crc32 = write_pack_object(f, type_num, data, sha=new_sha) + crc32 = write_pack_object( + f, type_num, data, sha=new_sha, + compression_level=self.pack_compression_level) entries.append((ext_sha, offset, crc32)) pack_sha = new_sha.digest() f.write(pack_sha) @@ -693,9 +723,8 @@ def _complete_thin_pack(self, f, path, copier, indexer): # removal, silently passing if the target does not exist. try: os.remove(target_pack) - except (IOError, OSError) as e: - if e.errno != errno.ENOENT: - raise + except FileNotFoundError: + pass os.rename(path, target_pack) # Write the index. @@ -727,6 +756,7 @@ def add_thin_pack(self, read_all, read_some): Returns: A Pack object pointing at the now-completed thin pack in the objects/pack directory. """ + import tempfile fd, path = tempfile.mkstemp(dir=self.path, prefix='tmp_pack_') with os.fdopen(fd, 'w+b') as f: indexer = PackIndexer(f, resolve_ext_ref=self.get_raw) @@ -760,9 +790,8 @@ def move_in_pack(self, path): # removal, silently passing if the target does not exist. try: os.remove(target_pack) - except (IOError, OSError) as e: - if e.errno != errno.ENOENT: - raise + except FileNotFoundError: + pass os.rename(path, target_pack) final_pack = Pack(basename) self._add_cached_pack(basename, final_pack) @@ -775,6 +804,7 @@ def add_pack(self): call when the pack is finished and an abort function. """ + import tempfile fd, path = tempfile.mkstemp(dir=self.pack_dir, suffix=".pack") f = os.fdopen(fd, 'wb') @@ -803,21 +833,20 @@ def add_object(self, obj): dir = os.path.dirname(path) try: os.mkdir(dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise + except FileExistsError: + pass if os.path.exists(path): return # Already there, no need to write again with GitFile(path, 'wb') as f: - f.write(obj.as_legacy_object()) + f.write(obj.as_legacy_object( + compression_level=self.loose_compression_level)) @classmethod def init(cls, path): try: os.mkdir(path) - except OSError as e: - if e.errno != errno.EEXIST: - raise + except FileExistsError: + pass os.mkdir(os.path.join(path, "info")) os.mkdir(os.path.join(path, PACKDIR)) return cls(path) @@ -829,6 +858,7 @@ class MemoryObjectStore(BaseObjectStore): def __init__(self): super(MemoryObjectStore, self).__init__() self._data = {} + self.pack_compression_level = -1 def _to_hexsha(self, sha): if len(sha) == 40: @@ -928,7 +958,8 @@ def _complete_thin_pack(self, f, indexer): for ext_sha in indexer.ext_refs(): assert len(ext_sha) == 20 type_num, data = self.get_raw(ext_sha) - write_pack_object(f, type_num, data, sha=new_sha) + write_pack_object( + f, type_num, data, sha=new_sha) pack_sha = new_sha.digest() f.write(pack_sha) @@ -1130,9 +1161,11 @@ class MissingObjectFinder(object): tagged: dict of pointed-to sha -> tag sha for including tags """ - def __init__(self, object_store, haves, wants, progress=None, + def __init__(self, object_store, haves, wants, shallow=None, progress=None, get_tagged=None, get_parents=lambda commit: commit.parents): self.object_store = object_store + if shallow is None: + shallow = set() self._get_parents = get_parents # process Commits and Tags differently # Note, while haves may list commits/tags not available locally, @@ -1146,12 +1179,13 @@ def __init__(self, object_store, haves, wants, progress=None, # all_ancestors is a set of commits that shall not be sent # (complete repository up to 'haves') all_ancestors = object_store._collect_ancestors( - have_commits, get_parents=self._get_parents)[0] + have_commits, shallow=shallow, get_parents=self._get_parents)[0] # all_missing - complete set of commits between haves and wants # common - commits from all_ancestors we hit into while # traversing parent hierarchy of wants missing_commits, common_commits = object_store._collect_ancestors( - want_commits, all_ancestors, get_parents=self._get_parents) + want_commits, all_ancestors, shallow=shallow, + get_parents=self._get_parents) self.sha_done = set() # Now, fill sha_done with commits and revisions of # files and directories known to be both locally @@ -1382,4 +1416,4 @@ def read_packs_file(f): (kind, name) = line.split(b" ", 1) if kind != b"P": continue - yield name.decode(sys.getfilesystemencoding()) + yield os.fsdecode(name) diff --git a/ext_libs/dulwich/objects.py b/ext_libs/dulwich/objects.py index 8976d4fd..2361feee 100644 --- a/ext_libs/dulwich/objects.py +++ b/ext_libs/dulwich/objects.py @@ -27,7 +27,12 @@ import os import posixpath import stat -import sys +from typing import ( + Optional, + Dict, + Union, + Type, + ) import warnings import zlib from hashlib import sha1 @@ -39,7 +44,7 @@ NotTagError, NotTreeError, ObjectFormatException, - EmptyFileException, + FileFormatException, ) from dulwich.file import GitFile @@ -70,6 +75,10 @@ BEGIN_PGP_SIGNATURE = b"-----BEGIN PGP SIGNATURE-----" +class EmptyFileException(FileFormatException): + """An unexpectedly empty file was encountered.""" + + def S_ISGITLINK(m): """Check if a mode indicates a submodule. @@ -142,13 +151,13 @@ def filename_to_hex(filename): return hex -def object_header(num_type, length): +def object_header(num_type: int, length: int) -> bytes: """Return an object header for the given numeric type and text length.""" return (object_class(num_type).type_name + b' ' + str(length).encode('ascii') + b'\0') -def serializable_property(name, docstring=None): +def serializable_property(name: str, docstring: Optional[str] = None): """A property that helps tracking whether serialization is necessary. """ def set(obj, value): @@ -249,6 +258,9 @@ class ShaFile(object): __slots__ = ('_chunked_text', '_sha', '_needs_serialization') + type_name = None # type: bytes + type_num = None # type: int + @staticmethod def _parse_legacy_object_header(magic, f): """Parse a legacy object, creating it but not reading the file.""" @@ -282,21 +294,22 @@ def _parse_legacy_object(self, map): raise ObjectFormatException("Invalid object header, no \\0") self.set_raw_string(text[header_end+1:]) - def as_legacy_object_chunks(self): + def as_legacy_object_chunks(self, compression_level=-1): """Return chunks representing the object in the experimental format. Returns: List of strings """ - compobj = zlib.compressobj() + compobj = zlib.compressobj(compression_level) yield compobj.compress(self._header()) for chunk in self.as_raw_chunks(): yield compobj.compress(chunk) yield compobj.flush() - def as_legacy_object(self): + def as_legacy_object(self, compression_level=-1): """Return string representing the object in the experimental format. """ - return b''.join(self.as_legacy_object_chunks()) + return b''.join(self.as_legacy_object_chunks( + compression_level=compression_level)) def as_raw_chunks(self): """Return chunks with serialization of the object. @@ -316,14 +329,9 @@ def as_raw_string(self): """ return b''.join(self.as_raw_chunks()) - if sys.version_info[0] >= 3: - def __bytes__(self): - """Return raw string serialization of this object.""" - return self.as_raw_string() - else: - def __str__(self): - """Return raw string serialization of this object.""" - return self.as_raw_string() + def __bytes__(self): + """Return raw string serialization of this object.""" + return self.as_raw_string() def __hash__(self): """Return unique hash for this object.""" @@ -586,7 +594,7 @@ def _set_data(self, data): self.set_raw_string(data) data = property(_get_data, _set_data, - "The text contained within the blob object.") + doc="The text contained within the blob object.") def _get_chunked(self): return self._chunked_text @@ -602,7 +610,7 @@ def _deserialize(self, chunks): chunked = property( _get_chunked, _set_chunked, - "The text within the blob object, as chunks (not necessarily lines).") + doc="The text in the blob object, as chunks (not necessarily lines)") @classmethod def from_path(cls, path): @@ -1417,7 +1425,7 @@ def _get_extra(self): Tag, ) -_TYPE_MAP = {} +_TYPE_MAP = {} # type: Dict[Union[bytes, int], Type[ShaFile]] for cls in OBJECT_CLASSES: _TYPE_MAP[cls.type_name] = cls @@ -1429,6 +1437,6 @@ def _get_extra(self): _sorted_tree_items_py = sorted_tree_items try: # Try to import C versions - from dulwich._objects import parse_tree, sorted_tree_items + from dulwich._objects import parse_tree, sorted_tree_items # type: ignore except ImportError: pass diff --git a/ext_libs/dulwich/objectspec.py b/ext_libs/dulwich/objectspec.py index f588d313..6446765a 100644 --- a/ext_libs/dulwich/objectspec.py +++ b/ext_libs/dulwich/objectspec.py @@ -83,7 +83,7 @@ def parse_ref(container, refspec): raise KeyError(refspec) -def parse_reftuple(lh_container, rh_container, refspec): +def parse_reftuple(lh_container, rh_container, refspec, force=False): """Parse a reftuple spec. Args: @@ -98,8 +98,6 @@ def parse_reftuple(lh_container, rh_container, refspec): if refspec.startswith(b"+"): force = True refspec = refspec[1:] - else: - force = False if b":" in refspec: (lh, rh) = refspec.split(b":") else: @@ -120,13 +118,15 @@ def parse_reftuple(lh_container, rh_container, refspec): return (lh, rh, force) -def parse_reftuples(lh_container, rh_container, refspecs): +def parse_reftuples( + lh_container, rh_container, refspecs, force=False): """Parse a list of reftuple specs to a list of reftuples. Args: lh_container: A RefsContainer object hh_container: A RefsContainer object refspecs: A list of refspecs or a string + force: Force overwriting for all reftuples Returns: A list of refs Raises: KeyError: If one of the refs can not be found @@ -136,7 +136,8 @@ def parse_reftuples(lh_container, rh_container, refspecs): ret = [] # TODO: Support * in refspecs for refspec in refspecs: - ret.append(parse_reftuple(lh_container, rh_container, refspec)) + ret.append(parse_reftuple( + lh_container, rh_container, refspec, force=force)) return ret diff --git a/ext_libs/dulwich/pack.py b/ext_libs/dulwich/pack.py index 7b276c36..6f475473 100644 --- a/ext_libs/dulwich/pack.py +++ b/ext_libs/dulwich/pack.py @@ -43,12 +43,6 @@ import struct from itertools import chain -try: - from itertools import imap, izip -except ImportError: - # Python3 - imap = map - izip = zip import os import sys @@ -363,8 +357,8 @@ def __eq__(self, other): if not isinstance(other, PackIndex): return False - for (name1, _, _), (name2, _, _) in izip(self.iterentries(), - other.iterentries()): + for (name1, _, _), (name2, _, _) in zip(self.iterentries(), + other.iterentries()): if name1 != name2: return False return True @@ -378,7 +372,7 @@ def __len__(self): def __iter__(self): """Iterate over the SHAs in this pack.""" - return imap(sha_to_hex, self._itersha()) + return map(sha_to_hex, self._itersha()) def iterentries(self): """Iterate over the entries in this pack index. @@ -710,7 +704,7 @@ def chunks_length(chunks): if isinstance(chunks, bytes): return len(chunks) else: - return sum(imap(len, chunks)) + return sum(map(len, chunks)) def unpack_object(read_all, read_some=None, compute_crc32=False, @@ -1531,13 +1525,14 @@ def pack_object_header(type_num, delta_base, size): return bytearray(header) -def write_pack_object(f, type, object, sha=None): +def write_pack_object(f, type, object, sha=None, compression_level=-1): """Write pack object to a file. Args: f: File to write to type: Numeric type of the object object: Object to write + compression_level: the zlib compression level Returns: Tuple with offset at which the object was written, and crc32 """ if type in DELTA_TYPES: @@ -1545,7 +1540,7 @@ def write_pack_object(f, type, object, sha=None): else: delta_base = None header = bytes(pack_object_header(type, delta_base, len(object))) - comp_data = zlib.compress(object) + comp_data = zlib.compress(object, compression_level) crc32 = 0 for data in (header, comp_data): f.write(data) @@ -1555,7 +1550,8 @@ def write_pack_object(f, type, object, sha=None): return crc32 & 0xffffffff -def write_pack(filename, objects, deltify=None, delta_window_size=None): +def write_pack(filename, objects, deltify=None, delta_window_size=None, + compression_level=-1): """Write a new pack data file. Args: @@ -1564,11 +1560,13 @@ def write_pack(filename, objects, deltify=None, delta_window_size=None): Should provide __len__ window_size: Delta window size deltify: Whether to deltify pack objects + compression_level: the zlib compression level Returns: Tuple with checksum of pack file and index file """ with GitFile(filename + '.pack', 'wb') as f: entries, data_sum = write_pack_objects( - f, objects, delta_window_size=delta_window_size, deltify=deltify) + f, objects, delta_window_size=delta_window_size, deltify=deltify, + compression_level=compression_level) entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()]) with GitFile(filename + '.idx', 'wb') as f: return data_sum, write_pack_index_v2(f, entries, data_sum) @@ -1632,7 +1630,8 @@ def pack_objects_to_data(objects): for (o, path) in objects)) -def write_pack_objects(f, objects, delta_window_size=None, deltify=None): +def write_pack_objects(f, objects, delta_window_size=None, deltify=None, + compression_level=-1): """Write a new pack data file. Args: @@ -1642,6 +1641,7 @@ def write_pack_objects(f, objects, delta_window_size=None, deltify=None): window_size: Sliding window size for searching for deltas; Set to None for default window size. deltify: Whether to deltify objects + compression_level: the zlib compression level to use Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum """ if deltify is None: @@ -1654,10 +1654,13 @@ def write_pack_objects(f, objects, delta_window_size=None, deltify=None): else: pack_contents_count, pack_contents = pack_objects_to_data(objects) - return write_pack_data(f, pack_contents_count, pack_contents) + return write_pack_data( + f, pack_contents_count, pack_contents, + compression_level=compression_level) -def write_pack_data(f, num_records, records, progress=None): +def write_pack_data( + f, num_records, records, progress=None, compression_level=-1): """Write a new pack data file. Args: @@ -1665,6 +1668,7 @@ def write_pack_data(f, num_records, records, progress=None): num_records: Number of records records: Iterator over type_num, object_id, delta_base, raw progress: Function to report progress to + compression_level: the zlib compression level Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum """ # Write the pack @@ -1686,7 +1690,8 @@ def write_pack_data(f, num_records, records, progress=None): else: type_num = OFS_DELTA raw = (offset - base_offset, raw) - crc32 = write_pack_object(f, type_num, raw) + crc32 = write_pack_object( + f, type_num, raw, compression_level=compression_level) entries[object_id] = (offset, crc32) return entries, f.write_sha() @@ -1917,20 +1922,24 @@ def __init__(self, basename, resolve_ext_ref=None): self.resolve_ext_ref = resolve_ext_ref @classmethod - def from_lazy_objects(self, data_fn, idx_fn): + def from_lazy_objects(cls, data_fn, idx_fn): """Create a new pack object from callables to load pack data and index objects.""" - ret = Pack('') + ret = cls('') ret._data_load = data_fn ret._idx_load = idx_fn return ret @classmethod - def from_objects(self, data, idx): + def from_objects(cls, data, idx): """Create a new pack object from pack data and index objects.""" - ret = Pack('') - ret._data_load = lambda: data - ret._idx_load = lambda: idx + ret = cls('') + ret._data = data + ret._data.pack = ret + ret._data_load = None + ret._idx = idx + ret._idx_load = None + ret.check_length_and_checksum() return ret def name(self): @@ -2083,6 +2092,9 @@ def keep(self, msg=None): try: - from dulwich._pack import apply_delta, bisect_find_sha # noqa: F811 + from dulwich._pack import ( # type: ignore # noqa: F811 + apply_delta, + bisect_find_sha, + ) except ImportError: pass diff --git a/ext_libs/dulwich/patch.py b/ext_libs/dulwich/patch.py index 3cc4eba2..a6c3e01d 100644 --- a/ext_libs/dulwich/patch.py +++ b/ext_libs/dulwich/patch.py @@ -104,7 +104,8 @@ def _format_range_unified(start, stop): def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', - tofiledate='', n=3, lineterm='\n'): + tofiledate='', n=3, lineterm='\n', tree_encoding='utf-8', + output_encoding='utf-8'): """difflib.unified_diff that can detect "No newline at end of file" as original "git diff" does. @@ -117,15 +118,15 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' todate = '\t{}'.format(tofiledate) if tofiledate else '' yield '--- {}{}{}'.format( - fromfile.decode("ascii"), + fromfile.decode(tree_encoding), fromdate, lineterm - ).encode('ascii') + ).encode(output_encoding) yield '+++ {}{}{}'.format( - tofile.decode("ascii"), + tofile.decode(tree_encoding), todate, lineterm - ).encode('ascii') + ).encode(output_encoding) first, last = group[0], group[-1] file1_range = _format_range_unified(first[1], last[2]) @@ -134,7 +135,7 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', file1_range, file2_range, lineterm - ).encode('ascii') + ).encode(output_encoding) for tag, i1, i2, j1, j2 in group: if tag == 'equal': diff --git a/ext_libs/dulwich/porcelain.py b/ext_libs/dulwich/porcelain.py index ccb6c155..060a3dd8 100644 --- a/ext_libs/dulwich/porcelain.py +++ b/ext_libs/dulwich/porcelain.py @@ -64,11 +64,18 @@ from io import BytesIO, RawIOBase import datetime import os +from pathlib import Path import posixpath import shutil import stat import sys import time +from typing import ( + Dict, + Optional, + Tuple, + Union, + ) from dulwich.archive import ( tar_stream, @@ -89,7 +96,9 @@ ) from dulwich.errors import ( SendPackError, - UpdateRefsError, + ) +from dulwich.graph import ( + can_fast_forward, ) from dulwich.ignore import IgnoreFilterManager from dulwich.index import ( @@ -126,6 +135,7 @@ ANNOTATED_TAG_SUFFIX, LOCAL_BRANCH_PREFIX, strip_peeled_refs, + RefsContainer, ) from dulwich.repo import (BaseRepo, Repo) from dulwich.server import ( @@ -156,20 +166,24 @@ def write(self, b): return None -if sys.version_info[0] == 2: - default_bytes_out_stream = sys.stdout or NoneStream() - default_bytes_err_stream = sys.stderr or NoneStream() -else: - default_bytes_out_stream = ( - getattr(sys.stdout, 'buffer', None) or NoneStream()) - default_bytes_err_stream = ( - getattr(sys.stderr, 'buffer', None) or NoneStream()) +default_bytes_out_stream = ( + getattr(sys.stdout, 'buffer', None) or NoneStream()) +default_bytes_err_stream = ( + getattr(sys.stderr, 'buffer', None) or NoneStream()) DEFAULT_ENCODING = 'utf-8' -class RemoteExists(Exception): +class Error(Exception): + """Porcelain-based error. """ + + def __init__(self, msg, inner=None): + super(Error, self).__init__(msg) + self.inner = inner + + +class RemoteExists(Error): """Raised when the remote already exists.""" @@ -196,7 +210,7 @@ def open_repo_closing(path_or_repo): return closing(Repo(path_or_repo)) -def path_to_tree_path(repopath, path): +def path_to_tree_path(repopath, path, tree_encoding=DEFAULT_ENCODING): """Convert a path to a path usable in an index, e.g. bytes and relative to the repository root. @@ -205,16 +219,62 @@ def path_to_tree_path(repopath, path): path: A path, absolute or relative to the cwd Returns: A path formatted for use in e.g. an index """ - if not isinstance(path, bytes): - path = path.encode(sys.getfilesystemencoding()) - if not isinstance(repopath, bytes): - repopath = repopath.encode(sys.getfilesystemencoding()) - treepath = os.path.relpath(path, repopath) - if treepath.startswith(b'..'): - raise ValueError('Path not in repo') - if os.path.sep != '/': - treepath = treepath.replace(os.path.sep.encode('ascii'), b'/') - return treepath + # Pathlib resolve before Python 3.6 could raises FileNotFoundError in case + # there is no file matching the path so we reuse the old implementation for + # Python 3.5 + if sys.version_info < (3, 6): + if not isinstance(path, bytes): + path = os.fsencode(path) + if not isinstance(repopath, bytes): + repopath = os.fsencode(repopath) + treepath = os.path.relpath(path, repopath) + if treepath.startswith(b'..'): + err_msg = 'Path %r not in repo path (%r)' % (path, repopath) + raise ValueError(err_msg) + if os.path.sep != '/': + treepath = treepath.replace(os.path.sep.encode('ascii'), b'/') + return treepath + else: + # Resolve might returns a relative path on Windows + # https://bugs.python.org/issue38671 + if sys.platform == 'win32': + path = os.path.abspath(path) + + path = Path(path).resolve() + + # Resolve and abspath seems to behave differently regarding symlinks, + # as we are doing abspath on the file path, we need to do the same on + # the repo path or they might not match + if sys.platform == 'win32': + repopath = os.path.abspath(repopath) + + repopath = Path(repopath).resolve() + + relpath = path.relative_to(repopath) + if sys.platform == 'win32': + return str(relpath).replace(os.path.sep, '/').encode(tree_encoding) + else: + return bytes(relpath) + + +class DivergedBranches(Error): + """Branches have diverged and fast-forward is not possible.""" + + +def check_diverged(repo, current_sha, new_sha): + """Check if updating to a sha can be done with fast forwarding. + + Args: + repo: Repository object + current_sha: Current head sha + new_sha: New head sha + """ + try: + can = can_fast_forward(repo, current_sha, new_sha) + except KeyError: + can = False + if not can: + raise DivergedBranches(current_sha, new_sha) def archive(repo, committish=None, outstream=default_bytes_out_stream, @@ -259,7 +319,7 @@ def symbolic_ref(repo, ref_name, force=False): with open_repo_closing(repo) as repo_obj: ref_path = _make_branch_ref(ref_name) if not force and ref_path not in repo_obj.refs.keys(): - raise ValueError('fatal: ref `%s` is not a ref' % ref_name) + raise Error('fatal: ref `%s` is not a ref' % ref_name) repo_obj.refs.set_symbolic_ref(b'HEAD', ref_path) @@ -345,7 +405,7 @@ def clone(source, target=None, bare=False, checkout=None, if checkout is None: checkout = (not bare) if checkout and bare: - raise ValueError("checkout and bare are incompatible") + raise Error("checkout and bare are incompatible") if target is None: target = source.split("/")[-1] @@ -360,9 +420,6 @@ def clone(source, target=None, bare=False, checkout=None, reflog_message = b'clone: from ' + source.encode('utf-8') try: - fetch_result = fetch( - r, source, origin, errstream=errstream, message=reflog_message, - depth=depth, **kwargs) target_config = r.get_config() if not isinstance(source, bytes): source = source.encode(DEFAULT_ENCODING) @@ -371,10 +428,13 @@ def clone(source, target=None, bare=False, checkout=None, (b'remote', origin), b'fetch', b'+refs/heads/*:refs/remotes/' + origin + b'/*') target_config.write_to_path() + fetch_result = fetch( + r, origin, errstream=errstream, message=reflog_message, + depth=depth, **kwargs) # TODO(jelmer): Support symref capability, # https://github.com/jelmer/dulwich/issues/485 try: - head = r[fetch_result[b'HEAD']] + head = r[fetch_result.refs[b'HEAD']] except KeyError: head = None else: @@ -400,17 +460,18 @@ def add(repo=".", paths=None): """ ignored = set() with open_repo_closing(repo) as r: + repo_path = Path(r.path).resolve() ignore_manager = IgnoreFilterManager.from_repo(r) if not paths: paths = list( - get_untracked_paths(os.getcwd(), r.path, r.open_index())) + get_untracked_paths( + str(Path(os.getcwd()).resolve()), + str(repo_path), r.open_index())) relpaths = [] if not isinstance(paths, list): paths = [paths] for p in paths: - relpath = os.path.relpath(p, r.path) - if relpath.startswith('..' + os.path.sep): - raise ValueError('path %r is not in repo' % relpath) + relpath = str(Path(p).resolve().relative_to(repo_path)) # FIXME: Support patterns, directories. if ignore_manager.is_ignored(relpath): ignored.add(relpath) @@ -446,7 +507,14 @@ def clean(repo=".", target_dir=None): with open_repo_closing(repo) as r: if not _is_subdir(target_dir, r.path): - raise ValueError("target_dir must be in the repo's working dir") + raise Error("target_dir must be in the repo's working dir") + + config = r.get_config_stack() + require_force = config.get_boolean( # noqa: F841 + (b'clean',), b'requireForce', True) + + # TODO(jelmer): if require_force is set, then make sure that -f, -i or + # -n is specified. index = r.open_index() ignore_manager = IgnoreFilterManager.from_repo(r) @@ -482,12 +550,12 @@ def remove(repo=".", paths=None, cached=False): with open_repo_closing(repo) as r: index = r.open_index() for p in paths: - full_path = os.path.abspath(p).encode(sys.getfilesystemencoding()) + full_path = os.fsencode(os.path.abspath(p)) tree_path = path_to_tree_path(r.path, p) try: index_sha = index[tree_path].sha except KeyError: - raise Exception('%s did not match any files' % p) + raise Error('%s did not match any files' % p) if not cached: try: @@ -507,12 +575,12 @@ def remove(repo=".", paths=None, cached=False): committed_sha = None if blob.id != index_sha and index_sha != committed_sha: - raise Exception( + raise Error( 'file has staged content differing ' 'from both the file and head: %s' % p) if index_sha != committed_sha: - raise Exception( + raise Error( 'file has staged changes: %s' % p) os.remove(full_path) del index[tree_path] @@ -703,7 +771,7 @@ def decode(x): print_commit(entry.commit, decode, outstream) if name_status: outstream.writelines( - [l+'\n' for l in print_name_status(entry.changes())]) + [line+'\n' for line in print_name_status(entry.changes())]) # TODO(jelmer): better default for encoding? @@ -852,7 +920,7 @@ def tag_delete(repo, name): elif isinstance(name, list): names = name else: - raise TypeError("Unexpected tag name type %r" % name) + raise Error("Unexpected tag name type %r" % name) for name in names: del r.refs[_make_tag_ref(name)] @@ -867,16 +935,43 @@ def reset(repo, mode, treeish="HEAD"): """ if mode != "hard": - raise ValueError("hard is the only mode currently supported") + raise Error("hard is the only mode currently supported") with open_repo_closing(repo) as r: tree = parse_tree(r, treeish) r.reset_index(tree.id) -def push(repo, remote_location, refspecs, +def get_remote_repo( + repo: Repo, + remote_location: Optional[Union[str, bytes]] = None + ) -> Tuple[Optional[str], str]: + config = repo.get_config() + if remote_location is None: + remote_location = get_branch_remote(repo) + if isinstance(remote_location, str): + encoded_location = remote_location.encode() + else: + encoded_location = remote_location + + section = (b'remote', encoded_location) + + remote_name = None # type: Optional[str] + + if config.has_section(section): + remote_name = encoded_location.decode() + url = config.get(section, 'url') + encoded_location = url + else: + remote_name = None + + return (remote_name, encoded_location.decode()) + + +def push(repo, remote_location=None, refspecs=None, outstream=default_bytes_out_stream, - errstream=default_bytes_err_stream, **kwargs): + errstream=default_bytes_err_stream, + force=False, **kwargs): """Remote push with dulwich via dulwich.client Args: @@ -885,46 +980,73 @@ def push(repo, remote_location, refspecs, refspecs: Refs to push to remote outstream: A stream file to write output errstream: A stream file to write errors + force: Force overwriting refs """ # Open the repo with open_repo_closing(repo) as r: + (remote_name, remote_location) = get_remote_repo(r, remote_location) # Get the client and path client, path = get_transport_and_path( remote_location, config=r.get_config_stack(), **kwargs) selected_refs = [] + remote_changed_refs = {} def update_refs(refs): - selected_refs.extend(parse_reftuples(r.refs, refs, refspecs)) + selected_refs.extend(parse_reftuples( + r.refs, refs, refspecs, force=force)) new_refs = {} # TODO: Handle selected_refs == {None: None} - for (lh, rh, force) in selected_refs: + for (lh, rh, force_ref) in selected_refs: if lh is None: new_refs[rh] = ZERO_SHA + remote_changed_refs[rh] = None else: - new_refs[rh] = r.refs[lh] + try: + localsha = r.refs[lh] + except KeyError: + raise Error( + 'No valid ref %s in local repository' % lh) + if not force_ref and rh in refs: + check_diverged(r, refs[rh], localsha) + new_refs[rh] = localsha + remote_changed_refs[rh] = localsha return new_refs err_encoding = getattr(errstream, 'encoding', None) or DEFAULT_ENCODING - remote_location_bytes = client.get_url(path).encode(err_encoding) + remote_location = client.get_url(path) try: - client.send_pack( + result = client.send_pack( path, update_refs, - generate_pack_data=r.object_store.generate_pack_data, + generate_pack_data=r.generate_pack_data, progress=errstream.write) + except SendPackError as e: + raise Error( + "Push to " + remote_location + + " failed -> " + e.args[0].decode(), inner=e) + else: errstream.write( - b"Push to " + remote_location_bytes + b" successful.\n") - except (UpdateRefsError, SendPackError) as e: - errstream.write(b"Push to " + remote_location_bytes + - b" failed -> " + e.message.encode(err_encoding) + - b"\n") + b"Push to " + + remote_location.encode(err_encoding) + b" successful.\n") + + for ref, error in (result.ref_status or {}).items(): + if error is not None: + errstream.write( + b"Push of ref %s failed: %s\n" % + (ref, error.encode(err_encoding))) + else: + errstream.write(b'Ref %s updated\n' % ref) + + if remote_name is not None: + _import_remote_refs(r.refs, remote_name, remote_changed_refs) def pull(repo, remote_location=None, refspecs=None, outstream=default_bytes_out_stream, - errstream=default_bytes_err_stream, **kwargs): + errstream=default_bytes_err_stream, fast_forward=True, + force=False, **kwargs): """Pull from remote via dulwich.client Args: @@ -936,23 +1058,31 @@ def pull(repo, remote_location=None, refspecs=None, """ # Open the repo with open_repo_closing(repo) as r: - if remote_location is None: - # TODO(jelmer): Lookup 'remote' for current branch in config - raise NotImplementedError( - "looking up remote from branch config not supported yet") + (remote_name, remote_location) = get_remote_repo(r, remote_location) + if refspecs is None: refspecs = [b"HEAD"] selected_refs = [] def determine_wants(remote_refs): selected_refs.extend( - parse_reftuples(remote_refs, r.refs, refspecs)) - return [remote_refs[lh] for (lh, rh, force) in selected_refs] + parse_reftuples(remote_refs, r.refs, refspecs, force=force)) + return [ + remote_refs[lh] for (lh, rh, force_ref) in selected_refs + if remote_refs[lh] not in r.object_store] client, path = get_transport_and_path( remote_location, config=r.get_config_stack(), **kwargs) fetch_result = client.fetch( path, r, progress=errstream.write, determine_wants=determine_wants) - for (lh, rh, force) in selected_refs: + for (lh, rh, force_ref) in selected_refs: + try: + check_diverged( + r, r.refs[rh], fetch_result.refs[lh]) + except DivergedBranches: + if fast_forward: + raise + else: + raise NotImplementedError('merge is not yet supported') r.refs[rh] = fetch_result.refs[lh] if selected_refs: r[b'HEAD'] = fetch_result.refs[selected_refs[0][1]] @@ -960,6 +1090,8 @@ def determine_wants(remote_refs): # Perform 'git checkout .' - syncs staged changes tree = r[b"HEAD"].tree r.reset_index(tree=tree) + if remote_name is not None: + _import_remote_refs(r.refs, remote_name, fetch_result.refs) def status(repo=".", ignored=False): @@ -1066,7 +1198,7 @@ def get_tree_changes(repo): elif change[0][0] == change[0][1]: tracked_changes['modify'].append(change[0][0]) else: - raise AssertionError('git mv ops not yet supported') + raise NotImplementedError('git mv ops not yet supported') return tracked_changes @@ -1203,7 +1335,8 @@ def branch_create(repo, name, objectish=None, force=False): r.refs.set_if_equals(refname, None, object.id, message=ref_message) else: if not r.refs.add_if_new(refname, object.id, message=ref_message): - raise KeyError("Branch with name %s already exists." % name) + raise Error( + "Branch with name %s already exists." % name) def branch_list(repo): @@ -1234,15 +1367,55 @@ def active_branch(repo): return active_ref[len(LOCAL_BRANCH_PREFIX):] -def fetch(repo, remote_location, remote_name=b'origin', outstream=sys.stdout, - errstream=default_bytes_err_stream, message=None, depth=None, - prune=False, prune_tags=False, **kwargs): +def get_branch_remote(repo): + """Return the active branch's remote name, if any. + + Args: + repo: Repository to open + Returns: + remote name + Raises: + KeyError: if the repository does not have a working tree + """ + with open_repo_closing(repo) as r: + branch_name = active_branch(r.path) + config = r.get_config() + try: + remote_name = config.get((b'branch', branch_name), b'remote') + except KeyError: + remote_name = b'origin' + return remote_name + + +def _import_remote_refs( + refs_container: RefsContainer, remote_name: str, + refs: Dict[str, str], message: Optional[bytes] = None, + prune: bool = False, prune_tags: bool = False): + stripped_refs = strip_peeled_refs(refs) + branches = { + n[len(LOCAL_BRANCH_PREFIX):]: v for (n, v) in stripped_refs.items() + if n.startswith(LOCAL_BRANCH_PREFIX)} + refs_container.import_refs( + b'refs/remotes/' + remote_name.encode(), branches, message=message, + prune=prune) + tags = { + n[len(b'refs/tags/'):]: v for (n, v) in stripped_refs.items() + if n.startswith(b'refs/tags/') and + not n.endswith(ANNOTATED_TAG_SUFFIX)} + refs_container.import_refs( + b'refs/tags', tags, message=message, + prune=prune_tags) + + +def fetch(repo, remote_location=None, + outstream=sys.stdout, errstream=default_bytes_err_stream, + message=None, depth=None, prune=False, prune_tags=False, force=False, + **kwargs): """Fetch objects from a remote server. Args: repo: Path to the repository remote_location: String identifying a remote server - remote_name: Name for remote server outstream: Output stream (defaults to stdout) errstream: Error stream (defaults to stderr) message: Reflog message (defaults to b"fetch: from ") @@ -1252,28 +1425,19 @@ def fetch(repo, remote_location, remote_name=b'origin', outstream=sys.stdout, Returns: Dictionary with refs on the remote """ - if message is None: - message = b'fetch: from ' + remote_location.encode("utf-8") with open_repo_closing(repo) as r: + (remote_name, remote_location) = get_remote_repo(r, remote_location) + if message is None: + message = b'fetch: from ' + remote_location.encode("utf-8") client, path = get_transport_and_path( remote_location, config=r.get_config_stack(), **kwargs) fetch_result = client.fetch(path, r, progress=errstream.write, depth=depth) - stripped_refs = strip_peeled_refs(fetch_result.refs) - branches = { - n[len(LOCAL_BRANCH_PREFIX):]: v for (n, v) in stripped_refs.items() - if n.startswith(LOCAL_BRANCH_PREFIX)} - r.refs.import_refs( - b'refs/remotes/' + remote_name, branches, message=message, - prune=prune) - tags = { - n[len(b'refs/tags/'):]: v for (n, v) in stripped_refs.items() - if n.startswith(b'refs/tags/') and - not n.endswith(ANNOTATED_TAG_SUFFIX)} - r.refs.import_refs( - b'refs/tags', tags, message=message, - prune=prune_tags) - return fetch_result.refs + if remote_name is not None: + _import_remote_refs( + r.refs, remote_name, fetch_result.refs, message, prune=prune, + prune_tags=prune_tags) + return fetch_result def ls_remote(remote, config=None, **kwargs): @@ -1428,12 +1592,9 @@ def check_mailmap(repo, contact): """ with open_repo_closing(repo) as r: from dulwich.mailmap import Mailmap - import errno try: mailmap = Mailmap.from_path(os.path.join(r.path, '.mailmap')) - except IOError as e: - if e.errno != errno.ENOENT: - raise + except FileNotFoundError: mailmap = Mailmap() return mailmap.lookup(contact) diff --git a/ext_libs/dulwich/protocol.py b/ext_libs/dulwich/protocol.py index 756fe663..48ba16f0 100644 --- a/ext_libs/dulwich/protocol.py +++ b/ext_libs/dulwich/protocol.py @@ -48,6 +48,7 @@ # fatal error message just before stream aborts SIDE_BAND_CHANNEL_FATAL = 3 +CAPABILITY_ATOMIC = b'atomic' CAPABILITY_DEEPEN_SINCE = b'deepen-since' CAPABILITY_DEEPEN_NOT = b'deepen-not' CAPABILITY_DEEPEN_RELATIVE = b'deepen-relative' @@ -66,6 +67,8 @@ CAPABILITY_THIN_PACK = b'thin-pack' CAPABILITY_AGENT = b'agent' CAPABILITY_SYMREF = b'symref' +CAPABILITY_ALLOW_TIP_SHA1_IN_WANT = b'allow-tip-sha1-in-want' +CAPABILITY_ALLOW_REACHABLE_SHA1_IN_WANT = b'allow-reachable-sha1-in-want' # Magic ref that is used to attach capabilities to when # there are no refs. Should always be ste to ZERO_SHA. @@ -87,9 +90,15 @@ CAPABILITY_SHALLOW, CAPABILITY_DEEPEN_NOT, CAPABILITY_DEEPEN_RELATIVE, + CAPABILITY_ALLOW_TIP_SHA1_IN_WANT, + CAPABILITY_ALLOW_REACHABLE_SHA1_IN_WANT, ]) KNOWN_RECEIVE_CAPABILITIES = set(COMMON_CAPABILITIES + [ - CAPABILITY_REPORT_STATUS]) + CAPABILITY_REPORT_STATUS, + CAPABILITY_DELETE_REFS, + CAPABILITY_QUIET, + CAPABILITY_ATOMIC, + ]) def agent_string(): diff --git a/ext_libs/dulwich/reflog.py b/ext_libs/dulwich/reflog.py index 37a2ff8c..64cb6c5d 100644 --- a/ext_libs/dulwich/reflog.py +++ b/ext_libs/dulwich/reflog.py @@ -75,5 +75,5 @@ def read_reflog(f): f: File-like object Returns: Iterator over Entry objects """ - for l in f: - yield parse_reflog_line(l) + for line in f: + yield parse_reflog_line(line) diff --git a/ext_libs/dulwich/refs.py b/ext_libs/dulwich/refs.py index d597289d..5618f43f 100644 --- a/ext_libs/dulwich/refs.py +++ b/ext_libs/dulwich/refs.py @@ -22,9 +22,7 @@ """Ref handling. """ -import errno import os -import sys from dulwich.errors import ( PackedRefsException, @@ -148,15 +146,19 @@ def import_refs(self, base, other, committer=None, timestamp=None, else: to_delete = set() for name, value in other.items(): - self.set_if_equals(b'/'.join((base, name)), None, value, - message=message) + if value is None: + to_delete.add(name) + else: + self.set_if_equals(b'/'.join((base, name)), None, value, + message=message) if to_delete: try: to_delete.remove(name) except KeyError: pass for ref in to_delete: - self.remove_if_equals(b'/'.join((base, ref)), None) + self.remove_if_equals( + b'/'.join((base, ref)), None, message=message) def allkeys(self): """All refs present in this container.""" @@ -388,6 +390,35 @@ def get_symrefs(self): ret[src] = dst return ret + def watch(self): + """Watch for changes to the refs in this container. + + Returns a context manager that yields tuples with (refname, new_sha) + """ + raise NotImplementedError(self.watch) + + +class _DictRefsWatcher(object): + + def __init__(self, refs): + self._refs = refs + + def __enter__(self): + from queue import Queue + self.queue = Queue() + self._refs._watchers.add(self) + return self + + def __next__(self): + return self.queue.get() + + def _notify(self, entry): + self.queue.put_nowait(entry) + + def __exit__(self, exc_type, exc_val, exc_tb): + self._refs._watchers.remove(self) + return False + class DictRefsContainer(RefsContainer): """RefsContainer backed by a simple dict. @@ -400,6 +431,7 @@ def __init__(self, refs, logger=None): super(DictRefsContainer, self).__init__(logger=logger) self._refs = refs self._peeled = {} + self._watchers = set() def allkeys(self): return self._refs.keys() @@ -410,11 +442,20 @@ def read_loose_ref(self, name): def get_packed_refs(self): return {} + def _notify(self, ref, newsha): + for watcher in self._watchers: + watcher._notify((ref, newsha)) + + def watch(self): + return _DictRefsWatcher(self) + def set_symbolic_ref(self, name, other, committer=None, timestamp=None, timezone=None, message=None): old = self.follow(name)[-1] - self._refs[name] = SYMREF + other - self._log(name, old, old, committer=committer, timestamp=timestamp, + new = SYMREF + other + self._refs[name] = new + self._notify(name, new) + self._log(name, old, new, committer=committer, timestamp=timestamp, timezone=timezone, message=message) def set_if_equals(self, name, old_ref, new_ref, committer=None, @@ -426,6 +467,7 @@ def set_if_equals(self, name, old_ref, new_ref, committer=None, self._check_refname(realname) old = self._refs.get(realname) self._refs[realname] = new_ref + self._notify(realname, new_ref) self._log(realname, old, new_ref, committer=committer, timestamp=timestamp, timezone=timezone, message=message) return True @@ -435,6 +477,7 @@ def add_if_new(self, name, ref, committer=None, timestamp=None, if name in self._refs: return False self._refs[name] = ref + self._notify(name, ref) self._log(name, None, ref, committer=committer, timestamp=timestamp, timezone=timezone, message=message) return True @@ -448,6 +491,7 @@ def remove_if_equals(self, name, old_ref, committer=None, timestamp=None, except KeyError: pass else: + self._notify(name, None) self._log(name, old, None, committer=committer, timestamp=timestamp, timezone=timezone, message=message) return True @@ -459,7 +503,8 @@ def _update(self, refs): """Update multiple refs; intended only for testing.""" # TODO(dborowitz): replace this with a public function that uses # set_if_equal. - self._refs.update(refs) + for ref, sha in refs.items(): + self.set_if_equals(ref, None, sha) def _update_peeled(self, peeled): """Update cached peeled refs; intended only for testing.""" @@ -472,8 +517,8 @@ class InfoRefsContainer(RefsContainer): def __init__(self, f): self._refs = {} self._peeled = {} - for l in f.readlines(): - sha, name = l.rstrip(b'\n').split(b'\t') + for line in f.readlines(): + sha, name = line.rstrip(b'\n').split(b'\t') if name.endswith(ANNOTATED_TAG_SUFFIX): name = name[:-3] if not check_ref_format(name): @@ -500,18 +545,59 @@ def get_peeled(self, name): return self._refs[name] +class _InotifyRefsWatcher(object): + + def __init__(self, path): + import pyinotify + from queue import Queue + self.path = os.fsdecode(path) + self.manager = pyinotify.WatchManager() + self.manager.add_watch( + self.path, pyinotify.IN_DELETE | + pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO, rec=True, + auto_add=True) + + self.notifier = pyinotify.ThreadedNotifier( + self.manager, default_proc_fun=self._notify) + self.queue = Queue() + + def _notify(self, event): + if event.dir: + return + if event.pathname.endswith('.lock'): + return + ref = os.fsencode(os.path.relpath(event.pathname, self.path)) + if event.maskname == 'IN_DELETE': + self.queue.put_nowait((ref, None)) + elif event.maskname in ('IN_CLOSE_WRITE', 'IN_MOVED_TO'): + with open(event.pathname, 'rb') as f: + sha = f.readline().rstrip(b'\n\r') + self.queue.put_nowait((ref, sha)) + + def __next__(self): + return self.queue.get() + + def __enter__(self): + self.notifier.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.notifier.stop() + return False + + class DiskRefsContainer(RefsContainer): """Refs container that reads refs from disk.""" def __init__(self, path, worktree_path=None, logger=None): super(DiskRefsContainer, self).__init__(logger=logger) if getattr(path, 'encode', None) is not None: - path = path.encode(sys.getfilesystemencoding()) + path = os.fsencode(path) self.path = path if worktree_path is None: worktree_path = path if getattr(worktree_path, 'encode', None) is not None: - worktree_path = worktree_path.encode(sys.getfilesystemencoding()) + worktree_path = os.fsencode(worktree_path) self.worktree_path = worktree_path self._packed_refs = None self._peeled_refs = None @@ -525,8 +611,7 @@ def subkeys(self, base): for root, unused_dirs, files in os.walk(path): dir = root[len(path):] if os.path.sep != '/': - dir = dir.replace(os.path.sep.encode( - sys.getfilesystemencoding()), b"/") + dir = dir.replace(os.fsencode(os.path.sep), b"/") dir = dir.strip(b'/') for filename in files: refname = b"/".join(([dir] if dir else []) + [filename]) @@ -548,8 +633,7 @@ def allkeys(self): for root, unused_dirs, files in os.walk(refspath): dir = root[len(path):] if os.path.sep != '/': - dir = dir.replace( - os.path.sep.encode(sys.getfilesystemencoding()), b"/") + dir = dir.replace(os.fsencode(os.path.sep), b"/") for filename in files: refname = b"/".join([dir, filename]) if check_ref_format(refname): @@ -562,9 +646,7 @@ def refpath(self, name): """ if os.path.sep != "/": - name = name.replace( - b"/", - os.path.sep.encode(sys.getfilesystemencoding())) + name = name.replace(b"/", os.fsencode(os.path.sep)) # TODO: as the 'HEAD' reference is working tree specific, it # should actually not be a part of RefsContainer if name == b'HEAD': @@ -589,10 +671,8 @@ def get_packed_refs(self): path = os.path.join(self.path, b'packed-refs') try: f = GitFile(path, 'rb') - except IOError as e: - if e.errno == errno.ENOENT: - return {} - raise + except FileNotFoundError: + return {} with f: first_line = next(iter(f)).rstrip() if (first_line.startswith(b'# pack-refs') and b' peeled' in @@ -649,10 +729,8 @@ def read_loose_ref(self, name): else: # Read only the first 40 bytes return header + f.read(40 - len(SYMREF)) - except IOError as e: - if e.errno in (errno.ENOENT, errno.EISDIR, errno.ENOTDIR): - return None - raise + except (FileNotFoundError, IsADirectoryError, NotADirectoryError): + return None def _remove_packed_ref(self, name): if self._packed_refs is None: @@ -728,8 +806,7 @@ def set_if_equals(self, name, old_ref, new_ref, committer=None, packed_refs = self.get_packed_refs() while probe_ref: if packed_refs.get(probe_ref, None) is not None: - raise OSError(errno.ENOTDIR, - 'Not a directory: {}'.format(filename)) + raise NotADirectoryError(filename) probe_ref = os.path.dirname(probe_ref) ensure_dir_exists(os.path.dirname(filename)) @@ -823,9 +900,8 @@ def remove_if_equals(self, name, old_ref, committer=None, timestamp=None, # remove the reference file itself try: os.remove(filename) - except OSError as e: - if e.errno != errno.ENOENT: # may only be packed - raise + except FileNotFoundError: + pass # may only be packed self._remove_packed_ref(name) self._log(name, old_ref, None, committer=committer, @@ -856,6 +932,10 @@ def remove_if_equals(self, name, old_ref, committer=None, timestamp=None, return True + def watch(self): + import pyinotify # noqa: F401 + return _InotifyRefsWatcher(self.path) + def _split_ref_line(line): """Split a single ref line into a tuple of SHA1 and name.""" @@ -877,14 +957,14 @@ def read_packed_refs(f): f: file-like object to read from Returns: Iterator over tuples with SHA1s and ref names. """ - for l in f: - if l.startswith(b'#'): + for line in f: + if line.startswith(b'#'): # Comment continue - if l.startswith(b'^'): + if line.startswith(b'^'): raise PackedRefsException( "found peeled ref in packed-refs without peeled") - yield _split_ref_line(l) + yield _split_ref_line(line) def read_packed_refs_with_peeled(f): @@ -939,8 +1019,8 @@ def write_packed_refs(f, packed_refs, peeled_refs=None): def read_info_refs(f): ret = {} - for l in f.readlines(): - (sha, name) = l.rstrip(b"\r\n").split(b"\t", 1) + for line in f.readlines(): + (sha, name) = line.rstrip(b"\r\n").split(b"\t", 1) ret[name] = sha return ret diff --git a/ext_libs/dulwich/repo.py b/ext_libs/dulwich/repo.py index 1e1a4c42..f27efc9e 100644 --- a/ext_libs/dulwich/repo.py +++ b/ext_libs/dulwich/repo.py @@ -29,11 +29,18 @@ """ from io import BytesIO -import errno import os import sys import stat import time +from typing import Optional, Tuple, TYPE_CHECKING, List, Dict, Union, Iterable + +if TYPE_CHECKING: + # There are no circular imports here, but we try to defer imports as long + # as possible to reduce start-up time for anything that doesn't need + # these imports. + from dulwich.config import StackedConfig, ConfigFile + from dulwich.index import Index from dulwich.errors import ( NoIndexPresent, @@ -52,10 +59,12 @@ from dulwich.object_store import ( DiskObjectStore, MemoryObjectStore, + BaseObjectStore, ObjectStoreGraphWalker, ) from dulwich.objects import ( check_hexsha, + valid_hexsha, Blob, Commit, ShaFile, @@ -67,9 +76,11 @@ ) from dulwich.hooks import ( + Hook, PreCommitShellHook, PostCommitShellHook, CommitMsgShellHook, + PostReceiveShellHook, ) from dulwich.line_ending import BlobNormalizer @@ -120,7 +131,7 @@ def __init__(self, identity): self.identity = identity -def _get_default_identity(): +def _get_default_identity() -> Tuple[str, str]: import getpass import socket username = getpass.getuser() @@ -143,19 +154,38 @@ def _get_default_identity(): return (fullname, email) -def get_user_identity(config, kind=None): +def get_user_identity( + config: 'StackedConfig', + kind: Optional[str] = None) -> bytes: """Determine the identity to use for new commits. + + If kind is set, this first checks + GIT_${KIND}_NAME and GIT_${KIND}_EMAIL. + + If those variables are not set, then it will fall back + to reading the user.name and user.email settings from + the specified configuration. + + If that also fails, then it will fall back to using + the current users' identity as obtained from the host + system (e.g. the gecos field, $EMAIL, $USER@$(hostname -f). + + Args: + kind: Optional kind to return identity for, + usually either "AUTHOR" or "COMMITTER". + + Returns: + A user identity """ + user = None # type: Optional[bytes] + email = None # type: Optional[bytes] if kind: - user = os.environ.get("GIT_" + kind + "_NAME") - if user is not None: - user = user.encode('utf-8') - email = os.environ.get("GIT_" + kind + "_EMAIL") - if email is not None: - email = email.encode('utf-8') - else: - user = None - email = None + user_uc = os.environ.get("GIT_" + kind + "_NAME") + if user_uc is not None: + user = user_uc.encode('utf-8') + email_uc = os.environ.get("GIT_" + kind + "_EMAIL") + if email_uc is not None: + email = email_uc.encode('utf-8') if user is None: try: user = config.get(("user", ), "name") @@ -168,16 +198,12 @@ def get_user_identity(config, kind=None): email = None default_user, default_email = _get_default_identity() if user is None: - user = default_user - if not isinstance(user, bytes): - user = user.encode('utf-8') + user = default_user.encode('utf-8') if email is None: - email = default_email - if not isinstance(email, bytes): - email = email.encode('utf-8') + email = default_email.encode('utf-8') if email.startswith(b'<') and email.endswith(b'>'): email = email[1:-1] - return (user + b" <" + email + b">") + return user + b" <" + email + b">" def check_user_identity(identity): @@ -196,7 +222,8 @@ def check_user_identity(identity): raise InvalidUserIdentity(identity) -def parse_graftpoints(graftpoints): +def parse_graftpoints( + graftpoints: Iterable[bytes]) -> Dict[bytes, List[bytes]]: """Convert a list of graftpoints into a dict Args: @@ -211,8 +238,8 @@ def parse_graftpoints(graftpoints): https://git.wiki.kernel.org/index.php/GraftPoint """ grafts = {} - for l in graftpoints: - raw_graft = l.split(None, 1) + for line in graftpoints: + raw_graft = line.split(None, 1) commit = raw_graft[0] if len(raw_graft) == 2: @@ -227,7 +254,7 @@ def parse_graftpoints(graftpoints): return grafts -def serialize_graftpoints(graftpoints): +def serialize_graftpoints(graftpoints: Dict[bytes, List[bytes]]) -> bytes: """Convert a dictionary of grafts into string The graft dictionary is: @@ -263,13 +290,32 @@ def _set_filesystem_hidden(path): ("SetFileAttributesW", ctypes.windll.kernel32)) if isinstance(path, bytes): - path = path.decode(sys.getfilesystemencoding()) + path = os.fsdecode(path) if not SetFileAttributesW(path, FILE_ATTRIBUTE_HIDDEN): pass # Could raise or log `ctypes.WinError()` here # Could implement other platform specific filesytem hiding here +class ParentsProvider(object): + + def __init__(self, store, grafts={}, shallows=[]): + self.store = store + self.grafts = grafts + self.shallows = set(shallows) + + def get_parents(self, commit_id, commit=None): + try: + return self.grafts[commit_id] + except KeyError: + pass + if commit_id in self.shallows: + return [] + if commit is None: + commit = self.store[commit_id] + return commit.parents + + class BaseRepo(object): """Base class for a git repository. @@ -279,7 +325,7 @@ class BaseRepo(object): repository """ - def __init__(self, object_store, refs): + def __init__(self, object_store: BaseObjectStore, refs: RefsContainer): """Open a repository. This shouldn't be called directly, but rather through one of the @@ -292,17 +338,17 @@ def __init__(self, object_store, refs): self.object_store = object_store self.refs = refs - self._graftpoints = {} - self.hooks = {} + self._graftpoints = {} # type: Dict[bytes, List[bytes]] + self.hooks = {} # type: Dict[str, Hook] - def _determine_file_mode(self): + def _determine_file_mode(self) -> bool: """Probe the file-system to determine whether permissions can be trusted. Returns: True if permissions can be trusted, False otherwise. """ raise NotImplementedError(self._determine_file_mode) - def _init_files(self, bare): + def _init_files(self, bare: bool) -> None: """Initialize a default set of named files.""" from dulwich.config import ConfigFile self._put_named_file('description', b"Unnamed repository") @@ -460,17 +506,31 @@ def fetch_objects(self, determine_wants, graph_walker, progress, # commits aren't missing. haves = [] + parents_provider = ParentsProvider( + self.object_store, shallows=shallows) + def get_parents(commit): - if commit.id in shallows: - return [] - return self.get_parents(commit.id, commit) + return parents_provider.get_parents(commit.id, commit) return self.object_store.iter_shas( self.object_store.find_missing_objects( - haves, wants, progress, - get_tagged, + haves, wants, self.get_shallow(), + progress, get_tagged, get_parents=get_parents)) + def generate_pack_data(self, have, want, progress=None, ofs_delta=None): + """Generate pack data objects for a set of wants/haves. + + Args: + have: List of SHA1s of objects that should not be sent + want: List of SHA1s of objects that should be sent + ofs_delta: Whether OFS deltas can be included + progress: Optional progress reporting method + """ + return self.object_store.generate_pack_data( + have, want, shallow=self.get_shallow(), + progress=progress, ofs_delta=ofs_delta) + def get_graph_walker(self, heads=None): """Retrieve a graph walker. @@ -485,17 +545,18 @@ def get_graph_walker(self, heads=None): heads = [ sha for sha in self.refs.as_dict(b'refs/heads').values() if sha in self.object_store] + parents_provider = ParentsProvider(self.object_store) return ObjectStoreGraphWalker( - heads, self.get_parents, shallow=self.get_shallow()) + heads, parents_provider.get_parents, shallow=self.get_shallow()) - def get_refs(self): + def get_refs(self) -> Dict[bytes, bytes]: """Get dictionary with all refs. Returns: A ``dict`` mapping ref names to SHA1s """ return self.refs.as_dict() - def head(self): + def head(self) -> bytes: """Return the SHA1 pointed at by HEAD.""" return self.refs[b'HEAD'] @@ -516,7 +577,7 @@ def _get_object(self, sha, cls): ret.type_name, cls.type_name)) return ret - def get_object(self, sha): + def get_object(self, sha: bytes) -> ShaFile: """Retrieve the object with the specified SHA. Args: @@ -527,7 +588,12 @@ def get_object(self, sha): """ return self.object_store[sha] - def get_parents(self, sha, commit=None): + def parents_provider(self): + return ParentsProvider( + self.object_store, grafts=self._graftpoints, + shallows=self.get_shallow()) + + def get_parents(self, sha: bytes, commit: Commit = None) -> List[bytes]: """Retrieve the parents of a specific commit. If the specific commit is a graftpoint, the graft parents @@ -538,13 +604,7 @@ def get_parents(self, sha, commit=None): commit: Optional commit matching the sha Returns: List of parents """ - - try: - return self._graftpoints[sha] - except KeyError: - if commit is None: - commit = self[sha] - return commit.parents + return self.parents_provider().get_parents(sha, commit) def get_config(self): """Retrieve the config object. @@ -569,7 +629,7 @@ def set_description(self, description): """ raise NotImplementedError(self.set_description) - def get_config_stack(self): + def get_config_stack(self) -> 'StackedConfig': """Return a config stack for this repository. This stack accesses the configuration for both this repository @@ -591,7 +651,7 @@ def get_shallow(self): if f is None: return set() with f: - return set(l.strip() for l in f) + return set(line.strip() for line in f) def update_shallow(self, new_shallow, new_unshallow): """Update the list of shallow objects. @@ -682,18 +742,18 @@ def __getitem__(self, name): except RefFormatError: raise KeyError(name) - def __contains__(self, name): + def __contains__(self, name: bytes) -> bool: """Check if a specific Git object or ref is present. Args: name: Git object SHA1 or ref name """ - if len(name) in (20, 40): + if len(name) == 20 or (len(name) == 40 and valid_hexsha(name)): return name in self.object_store or name in self.refs else: return name in self.refs - def __setitem__(self, name, value): + def __setitem__(self, name: bytes, value: Union[ShaFile, bytes]): """Set a ref. Args: @@ -710,7 +770,7 @@ def __setitem__(self, name, value): else: raise ValueError(name) - def __delitem__(self, name): + def __delitem__(self, name: bytes): """Remove a ref. Args: @@ -721,13 +781,14 @@ def __delitem__(self, name): else: raise ValueError(name) - def _get_user_identity(self, config, kind=None): + def _get_user_identity( + self, config: 'StackedConfig', kind: str = None) -> bytes: """Determine the identity to use for new commits. """ # TODO(jelmer): Deprecate this function in favor of get_user_identity return get_user_identity(config) - def _add_graftpoints(self, updated_graftpoints): + def _add_graftpoints(self, updated_graftpoints: Dict[bytes, List[bytes]]): """Add or modify graftpoints Args: @@ -741,7 +802,7 @@ def _add_graftpoints(self, updated_graftpoints): self._graftpoints.update(updated_graftpoints) - def _remove_graftpoints(self, to_remove=[]): + def _remove_graftpoints(self, to_remove: List[bytes] = []) -> None: """Remove graftpoints Args: @@ -755,7 +816,7 @@ def _read_heads(self, name): if f is None: return [] with f: - return [l.strip() for l in f.readlines() if l.strip()] + return [line.strip() for line in f.readlines() if line.strip()] def do_commit(self, message=None, committer=None, author=None, commit_timestamp=None, @@ -764,10 +825,14 @@ def do_commit(self, message=None, committer=None, ref=b'HEAD', merge_heads=None): """Create a new commit. + If not specified, `committer` and `author` default to + get_user_identity(..., 'COMMITTER') + and get_user_identity(..., 'AUTHOR') respectively. + Args: message: Commit message committer: Committer fullname - author: Author fullname (defaults to committer) + author: Author fullname commit_timestamp: Commit timestamp (defaults to now) commit_timezone: Commit timestamp timezone (defaults to GMT) author_timestamp: Author timestamp (defaults to commit @@ -779,7 +844,9 @@ def do_commit(self, message=None, committer=None, encoding: Encoding ref: Optional ref to commit to (defaults to current branch) merge_heads: Merge heads (defaults to .git/MERGE_HEADS) - Returns: New commit SHA1 + + Returns: + New commit SHA1 """ import time c = Commit() @@ -929,13 +996,14 @@ def __init__(self, root): with commondir: self._commondir = os.path.join( self.controldir(), - commondir.read().rstrip(b"\r\n").decode( - sys.getfilesystemencoding())) + os.fsdecode(commondir.read().rstrip(b"\r\n"))) else: self._commondir = self._controldir self.path = root - object_store = DiskObjectStore( - os.path.join(self.commondir(), OBJECTDIR)) + config = self.get_config() + object_store = DiskObjectStore.from_config( + os.path.join(self.commondir(), OBJECTDIR), + config) refs = DiskRefsContainer(self.commondir(), self._controldir, logger=self._write_reflog) BaseRepo.__init__(self, object_store, refs) @@ -955,18 +1023,16 @@ def __init__(self, root): self.hooks['pre-commit'] = PreCommitShellHook(self.controldir()) self.hooks['commit-msg'] = CommitMsgShellHook(self.controldir()) self.hooks['post-commit'] = PostCommitShellHook(self.controldir()) + self.hooks['post-receive'] = PostReceiveShellHook(self.controldir()) def _write_reflog(self, ref, old_sha, new_sha, committer, timestamp, timezone, message): from .reflog import format_reflog_line - path = os.path.join( - self.controldir(), 'logs', - ref.decode(sys.getfilesystemencoding())) + path = os.path.join(self.controldir(), 'logs', os.fsdecode(ref)) try: os.makedirs(os.path.dirname(path)) - except OSError as e: - if e.errno != errno.EEXIST: - raise + except FileExistsError: + pass if committer is None: config = self.get_config_stack() committer = self._get_user_identity(config) @@ -1026,10 +1092,8 @@ def _determine_file_mode(self): st1 = os.lstat(fname) try: os.chmod(fname, st1.st_mode ^ stat.S_IXUSR) - except EnvironmentError as e: - if e.errno == errno.EPERM: - return False - raise + except PermissionError: + return False st2 = os.lstat(fname) os.unlink(fname) @@ -1053,10 +1117,8 @@ def _put_named_file(self, path, contents): def _del_named_file(self, path): try: os.unlink(os.path.join(self.controldir(), path)) - except (IOError, OSError) as e: - if e.errno == errno.ENOENT: - return - raise + except FileNotFoundError: + return def get_named_file(self, path, basedir=None): """Get a file from the control dir with a specific name. @@ -1078,16 +1140,14 @@ def get_named_file(self, path, basedir=None): path = path.lstrip(os.path.sep) try: return open(os.path.join(basedir, path), 'rb') - except (IOError, OSError) as e: - if e.errno == errno.ENOENT: - return None - raise + except FileNotFoundError: + return None def index_path(self): """Return path to the index file.""" return os.path.join(self.controldir(), INDEX_FILENAME) - def open_index(self): + def open_index(self) -> 'Index': """Open the index for this repository. Raises: @@ -1112,7 +1172,7 @@ def stage(self, fs_paths): fs_paths: List of paths, relative to the repository path """ - root_path_bytes = self.path.encode(sys.getfilesystemencoding()) + root_path_bytes = os.fsencode(self.path) if not isinstance(fs_paths, list): fs_paths = [fs_paths] @@ -1125,7 +1185,7 @@ def stage(self, fs_paths): blob_normalizer = self.get_blob_normalizer() for fs_path in fs_paths: if not isinstance(fs_path, bytes): - fs_path = fs_path.encode(sys.getfilesystemencoding()) + fs_path = os.fsencode(fs_path) if os.path.isabs(fs_path): raise ValueError( "path %r should be relative to " @@ -1141,16 +1201,17 @@ def stage(self, fs_paths): except KeyError: pass # already removed else: - if not stat.S_ISDIR(st.st_mode): - blob = blob_from_path_and_stat(full_path, st) - blob = blob_normalizer.checkin_normalize(blob, fs_path) - self.object_store.add_object(blob) - index[tree_path] = index_entry_from_stat(st, blob.id, 0) - else: + if (not stat.S_ISREG(st.st_mode) and + not stat.S_ISLNK(st.st_mode)): try: del index[tree_path] except KeyError: pass + else: + blob = blob_from_path_and_stat(full_path, st) + blob = blob_normalizer.checkin_normalize(blob, fs_path) + self.object_store.add_object(blob) + index[tree_path] = index_entry_from_stat(st, blob.id, 0) index.write() def clone(self, target_path, mkdir=True, bare=False, @@ -1174,7 +1235,7 @@ def clone(self, target_path, mkdir=True, bare=False, self.fetch(target) encoded_path = self.path if not isinstance(encoded_path, bytes): - encoded_path = encoded_path.encode(sys.getfilesystemencoding()) + encoded_path = os.fsencode(encoded_path) ref_message = b"clone: from " + encoded_path target.refs.import_refs( b'refs/remotes/' + origin, self.refs.as_dict(b'refs/heads'), @@ -1234,7 +1295,7 @@ def reset_index(self, tree=None): honor_filemode=honor_filemode, validate_path_element=validate_path_element) - def get_config(self): + def get_config(self) -> 'ConfigFile': """Retrieve the config object. Returns: `ConfigFile` object for the ``.git/config`` file. @@ -1243,9 +1304,7 @@ def get_config(self): path = os.path.join(self._controldir, 'config') try: return ConfigFile.from_path(path) - except (IOError, OSError) as e: - if e.errno != errno.ENOENT: - raise + except FileNotFoundError: ret = ConfigFile() ret.path = path return ret @@ -1259,9 +1318,7 @@ def get_description(self): try: with GitFile(path, 'rb') as f: return f.read() - except (IOError, OSError) as e: - if e.errno != errno.ENOENT: - raise + except FileNotFoundError: return None def __repr__(self): @@ -1323,21 +1380,17 @@ def _init_new_working_directory(cls, path, main_repo, identifier=None, worktree_controldir = os.path.join(main_worktreesdir, identifier) gitdirfile = os.path.join(path, CONTROLDIR) with open(gitdirfile, 'wb') as f: - f.write(b'gitdir: ' + - worktree_controldir.encode(sys.getfilesystemencoding()) + - b'\n') + f.write(b'gitdir: ' + os.fsencode(worktree_controldir) + b'\n') try: os.mkdir(main_worktreesdir) - except OSError as e: - if e.errno != errno.EEXIST: - raise + except FileExistsError: + pass try: os.mkdir(worktree_controldir) - except OSError as e: - if e.errno != errno.EEXIST: - raise + except FileExistsError: + pass with open(os.path.join(worktree_controldir, GITDIR), 'wb') as f: - f.write(gitdirfile.encode(sys.getfilesystemencoding()) + b'\n') + f.write(os.fsencode(gitdirfile) + b'\n') with open(os.path.join(worktree_controldir, COMMONDIR), 'wb') as f: f.write(b'../..\n') with open(os.path.join(worktree_controldir, 'HEAD'), 'wb') as f: diff --git a/ext_libs/dulwich/server.py b/ext_libs/dulwich/server.py index f5af537e..10e2bf36 100644 --- a/ext_libs/dulwich/server.py +++ b/ext_libs/dulwich/server.py @@ -47,18 +47,17 @@ import socket import sys import time +from typing import List, Tuple, Dict, Optional, Iterable import zlib -try: - import SocketServer -except ImportError: - import socketserver as SocketServer +import socketserver from dulwich.archive import tar_stream from dulwich.errors import ( ApplyDeltaError, ChecksumMismatch, GitProtocolError, + HookError, NotGitRepository, UnexpectedCommandError, ObjectFormatException, @@ -75,6 +74,7 @@ BufferedPktLineWriter, capability_agent, CAPABILITIES_REF, + CAPABILITY_AGENT, CAPABILITY_DELETE_REFS, CAPABILITY_INCLUDE_TAG, CAPABILITY_MULTI_ACK_DETAILED, @@ -114,6 +114,7 @@ write_info_refs, ) from dulwich.repo import ( + BaseRepo, Repo, ) @@ -146,7 +147,7 @@ class BackendRepo(object): object_store = None refs = None - def get_refs(self): + def get_refs(self) -> Dict[bytes, bytes]: """ Get all the refs in the repository @@ -154,7 +155,7 @@ def get_refs(self): """ raise NotImplementedError - def get_peeled(self, name): + def get_peeled(self, name: bytes) -> Optional[bytes]: """Return the cached peeled value of a ref, if available. Args: @@ -185,7 +186,7 @@ class DictBackend(Backend): def __init__(self, repos): self.repos = repos - def open_repository(self, path): + def open_repository(self, path: str) -> BaseRepo: logger.debug('Opening repository at %s', path) try: return self.repos[path] @@ -218,10 +219,10 @@ def open_repository(self, path): class Handler(object): """Smart protocol command handler base class.""" - def __init__(self, backend, proto, http_req=None): + def __init__(self, backend, proto, stateless_rpc=None): self.backend = backend self.proto = proto - self.http_req = http_req + self.stateless_rpc = stateless_rpc def handle(self): raise NotImplementedError(self.handle) @@ -230,8 +231,8 @@ def handle(self): class PackHandler(Handler): """Protocol handler for packs.""" - def __init__(self, backend, proto, http_req=None): - super(PackHandler, self).__init__(backend, proto, http_req) + def __init__(self, backend, proto, stateless_rpc=None): + super(PackHandler, self).__init__(backend, proto, stateless_rpc) self._client_capabilities = None # Flags needed for the no-done capability self._done_received = False @@ -242,51 +243,53 @@ def capability_line(cls, capabilities): return b"".join([b" " + c for c in capabilities]) @classmethod - def capabilities(cls): + def capabilities(cls) -> Iterable[bytes]: raise NotImplementedError(cls.capabilities) @classmethod - def innocuous_capabilities(cls): + def innocuous_capabilities(cls) -> Iterable[bytes]: return [CAPABILITY_INCLUDE_TAG, CAPABILITY_THIN_PACK, CAPABILITY_NO_PROGRESS, CAPABILITY_OFS_DELTA, capability_agent()] @classmethod - def required_capabilities(cls): + def required_capabilities(cls) -> Iterable[bytes]: """Return a list of capabilities that we require the client to have.""" return [] - def set_client_capabilities(self, caps): + def set_client_capabilities(self, caps: Iterable[bytes]) -> None: allowable_caps = set(self.innocuous_capabilities()) allowable_caps.update(self.capabilities()) for cap in caps: + if cap.startswith(CAPABILITY_AGENT + b'='): + continue if cap not in allowable_caps: - raise GitProtocolError('Client asked for capability %s that ' + raise GitProtocolError('Client asked for capability %r that ' 'was not advertised.' % cap) for cap in self.required_capabilities(): if cap not in caps: raise GitProtocolError('Client does not support required ' - 'capability %s.' % cap) + 'capability %r.' % cap) self._client_capabilities = set(caps) logger.info('Client capabilities: %s', caps) - def has_capability(self, cap): + def has_capability(self, cap: bytes) -> bool: if self._client_capabilities is None: - raise GitProtocolError('Server attempted to access capability %s ' + raise GitProtocolError('Server attempted to access capability %r ' 'before asking client' % cap) return cap in self._client_capabilities - def notify_done(self): + def notify_done(self) -> None: self._done_received = True class UploadPackHandler(PackHandler): """Protocol handler for uploading a pack to the client.""" - def __init__(self, backend, args, proto, http_req=None, + def __init__(self, backend, args, proto, stateless_rpc=None, advertise_refs=False): super(UploadPackHandler, self).__init__( - backend, proto, http_req=http_req) + backend, proto, stateless_rpc=stateless_rpc) self.repo = backend.open_repository(args[0]) self._graph_walker = None self.advertise_refs = advertise_refs @@ -352,8 +355,14 @@ def write(x): graph_walker = _ProtocolGraphWalker( self, self.repo.object_store, self.repo.get_peeled, self.repo.refs.get_symrefs) + wants = [] + + def wants_wrapper(refs): + wants.extend(graph_walker.determine_wants(refs)) + return wants + objects_iter = self.repo.fetch_objects( - graph_walker.determine_wants, graph_walker, self.progress, + wants_wrapper, graph_walker, self.progress, get_tagged=self.get_tagged) # Note the fact that client is only processing responses related @@ -367,7 +376,7 @@ def write(x): # with a graph walker with an implementation that talks over the # wire (which is this instance of this class) this will actually # iterate through everything and write things out to the wire. - if len(objects_iter) == 0: + if len(wants) == 0: return # The provided haves are processed, and it is safe to send side- @@ -530,7 +539,7 @@ def __init__(self, handler, object_store, get_peeled, get_symrefs): self.get_peeled = get_peeled self.get_symrefs = get_symrefs self.proto = handler.proto - self.http_req = handler.http_req + self.stateless_rpc = handler.stateless_rpc self.advertise_refs = handler.advertise_refs self._wants = [] self.shallow = set() @@ -545,7 +554,7 @@ def determine_wants(self, heads): """Determine the wants for a set of heads. The given heads are advertised to the client, who then specifies which - refs he wants using 'want' lines. This portion of the protocol is the + refs they want using 'want' lines. This portion of the protocol is the same regardless of ack type, and in fact is used to set the ack type of the ProtocolGraphWalker. @@ -561,7 +570,7 @@ def determine_wants(self, heads): """ symrefs = self.get_symrefs() values = set(heads.values()) - if self.advertise_refs or not self.http_req: + if self.advertise_refs or not self.stateless_rpc: for i, (ref, sha) in enumerate(sorted(heads.items())): try: peeled_sha = self.get_peeled(ref) @@ -610,7 +619,7 @@ def determine_wants(self, heads): self.unread_proto_line(command, sha) self._handle_shallow_request(want_revs) - if self.http_req and self.proto.eof(): + if self.stateless_rpc and self.proto.eof(): # The client may close the socket at this point, expecting a # flush-pkt from the server. We might be ready to send a packfile # at this point, so we need to explicitly short-circuit in this @@ -635,7 +644,7 @@ def reset(self): def next(self): if not self._cached: - if not self._impl and self.http_req: + if not self._impl and self.stateless_rpc: return None return next(self._impl) self._cache_index += 1 @@ -844,7 +853,7 @@ def next(self): if self.walker.all_wants_satisfied(self._common): self.walker.send_ack(self._common[-1], b'ready') self.walker.send_nak() - if self.walker.http_req: + if self.walker.stateless_rpc: # The HTTP version of this request a flush-pkt always # signifies an end of request, so we also return # nothing here as if we are done (but not really, as @@ -893,20 +902,22 @@ def handle_done(self, done_required, done_received): class ReceivePackHandler(PackHandler): """Protocol handler for downloading a pack from the client.""" - def __init__(self, backend, args, proto, http_req=None, + def __init__(self, backend, args, proto, stateless_rpc=None, advertise_refs=False): super(ReceivePackHandler, self).__init__( - backend, proto, http_req=http_req) + backend, proto, stateless_rpc=stateless_rpc) self.repo = backend.open_repository(args[0]) self.advertise_refs = advertise_refs @classmethod - def capabilities(cls): + def capabilities(cls) -> Iterable[bytes]: return [CAPABILITY_REPORT_STATUS, CAPABILITY_DELETE_REFS, CAPABILITY_QUIET, CAPABILITY_OFS_DELTA, CAPABILITY_SIDE_BAND_64K, CAPABILITY_NO_DONE] - def _apply_pack(self, refs): + def _apply_pack( + self, refs: List[Tuple[bytes, bytes, bytes]] + ) -> List[Tuple[bytes, bytes]]: all_exceptions = (IOError, OSError, ChecksumMismatch, ApplyDeltaError, AssertionError, socket.error, zlib.error, ObjectFormatException) @@ -925,7 +936,8 @@ def _apply_pack(self, refs): self.repo.object_store.add_thin_pack(self.proto.read, recv) status.append((b'unpack', b'ok')) except all_exceptions as e: - status.append((b'unpack', str(e).replace('\n', ''))) + status.append( + (b'unpack', str(e).replace('\n', '').encode('utf-8'))) # The pack may still have been moved in, but it may contain # broken objects. We trust a later GC to clean it up. else: @@ -956,7 +968,7 @@ def _apply_pack(self, refs): return status - def _report_status(self, status): + def _report_status(self, status: List[Tuple[bytes, bytes]]) -> None: if self.has_capability(CAPABILITY_SIDE_BAND_64K): writer = BufferedPktLineWriter( lambda d: self.proto.write_sideband(SIDE_BAND_CHANNEL_DATA, d)) @@ -981,8 +993,19 @@ def flush(): write(None) flush() - def handle(self): - if self.advertise_refs or not self.http_req: + def _on_post_receive(self, client_refs): + hook = self.repo.hooks.get('post-receive', None) + if not hook: + return + try: + output = hook.execute(client_refs) + if output: + self.proto.write_sideband(SIDE_BAND_CHANNEL_PROGRESS, output) + except HookError as err: + self.proto.write_sideband(SIDE_BAND_CHANNEL_FATAL, repr(err)) + + def handle(self) -> None: + if self.advertise_refs or not self.stateless_rpc: refs = sorted(self.repo.get_refs().items()) symrefs = sorted(self.repo.refs.get_symrefs().items()) @@ -1018,6 +1041,8 @@ def handle(self): # backend can now deal with this refs and read a pack using self.read status = self._apply_pack(client_refs) + self._on_post_receive(client_refs) + # when we have read all the pack from the client, send a status report # if the client asked for it if self.has_capability(CAPABILITY_REPORT_STATUS): @@ -1026,8 +1051,9 @@ def handle(self): class UploadArchiveHandler(Handler): - def __init__(self, backend, args, proto, http_req=None): - super(UploadArchiveHandler, self).__init__(backend, proto, http_req) + def __init__(self, backend, args, proto, stateless_rpc=None): + super(UploadArchiveHandler, self).__init__( + backend, proto, stateless_rpc) self.repo = backend.open_repository(args[0]) def handle(self): @@ -1071,11 +1097,11 @@ def write(x): } -class TCPGitRequestHandler(SocketServer.StreamRequestHandler): +class TCPGitRequestHandler(socketserver.StreamRequestHandler): def __init__(self, handlers, *args, **kwargs): self.handlers = handlers - SocketServer.StreamRequestHandler.__init__(self, *args, **kwargs) + socketserver.StreamRequestHandler.__init__(self, *args, **kwargs) def handle(self): proto = ReceivableProtocol(self.connection.recv, self.wfile.write) @@ -1089,10 +1115,10 @@ def handle(self): h.handle() -class TCPGitServer(SocketServer.TCPServer): +class TCPGitServer(socketserver.TCPServer): allow_reuse_address = True - serve = SocketServer.TCPServer.serve_forever + serve = socketserver.TCPServer.serve_forever def _make_handler(self, *args, **kwargs): return TCPGitRequestHandler(self.handlers, *args, **kwargs) @@ -1104,7 +1130,7 @@ def __init__(self, backend, listen_addr, port=TCP_GIT_PORT, handlers=None): self.backend = backend logger.info('Listening for TCP connections on %s:%d', listen_addr, port) - SocketServer.TCPServer.__init__(self, (listen_addr, port), + socketserver.TCPServer.__init__(self, (listen_addr, port), self._make_handler) def verify_request(self, request, client_address): @@ -1177,7 +1203,7 @@ def generate_objects_info_packs(repo): """Generate an index for for packs.""" for pack in repo.object_store.packs: yield ( - b'P ' + pack.data.filename.encode(sys.getfilesystemencoding()) + + b'P ' + os.fsencode(pack.data.filename) + b'\n') diff --git a/ext_libs/dulwich/stash.py b/ext_libs/dulwich/stash.py index fdf9e7bc..72390ad6 100644 --- a/ext_libs/dulwich/stash.py +++ b/ext_libs/dulwich/stash.py @@ -22,7 +22,6 @@ from __future__ import absolute_import -import errno import os from dulwich.file import GitFile @@ -52,10 +51,8 @@ def stashes(self): try: with GitFile(reflog_path, 'rb') as f: return reversed(list(read_reflog(f))) - except EnvironmentError as e: - if e.errno == errno.ENOENT: - return [] - raise + except FileNotFoundError: + return [] @classmethod def from_repo(cls, repo): diff --git a/ext_libs/dulwich/web.py b/ext_libs/dulwich/web.py index 3e4469e6..d4d6aa6b 100644 --- a/ext_libs/dulwich/web.py +++ b/ext_libs/dulwich/web.py @@ -29,6 +29,7 @@ import re import sys import time +from typing import List, Tuple, Optional from wsgiref.simple_server import ( WSGIRequestHandler, ServerHandler, @@ -36,10 +37,7 @@ make_server, ) -try: - from urlparse import parse_qs -except ImportError: - from urllib.parse import parse_qs +from urllib.parse import parse_qs from dulwich import log_utils @@ -47,6 +45,7 @@ ReceivableProtocol, ) from dulwich.repo import ( + BaseRepo, NotGitRepository, Repo, ) @@ -68,7 +67,7 @@ HTTP_ERROR = '500 Internal Server Error' -def date_time_string(timestamp=None): +def date_time_string(timestamp: Optional[float] = None) -> str: # From BaseHTTPRequestHandler.date_time_string in BaseHTTPServer.py in the # Python 2.6.5 standard library, following modifications: # - Made a global rather than an instance method. @@ -81,12 +80,12 @@ def date_time_string(timestamp=None): 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] if timestamp is None: timestamp = time.time() - year, month, day, hh, mm, ss, wd, y, z = time.gmtime(timestamp) + year, month, day, hh, mm, ss, wd = time.gmtime(timestamp)[:7] return '%s, %02d %3s %4d %02d:%02d:%02d GMD' % ( weekdays[wd], day, months[month], year, hh, mm, ss) -def url_prefix(mat): +def url_prefix(mat) -> str: """Extract the URL prefix from a regex match. Args: @@ -98,7 +97,7 @@ def url_prefix(mat): return '/' + mat.string[:mat.start()].strip('/') -def get_repo(backend, mat): +def get_repo(backend, mat) -> BaseRepo: """Get a Repo instance for the given backend and URL regex match.""" return backend.open_repository(url_prefix(mat)) @@ -191,7 +190,7 @@ def get_info_refs(req, backend, mat): HTTP_OK, 'application/x-%s-advertisement' % service) proto = ReceivableProtocol(BytesIO().read, write) handler = handler_cls(backend, [url_prefix(mat)], proto, - http_req=req, advertise_refs=True) + stateless_rpc=req, advertise_refs=True) handler.proto.write_pkt_line( b'# service=' + service.encode('ascii') + b'\n') handler.proto.write_pkt_line(None) @@ -253,7 +252,7 @@ def handle_service_request(req, backend, mat): proto = ReceivableProtocol(req.environ['wsgi.input'].read, write) # TODO(jelmer): Find a way to pass in repo, rather than having handler_cls # reopen. - handler = handler_cls(backend, [url_prefix(mat)], proto, http_req=req) + handler = handler_cls(backend, [url_prefix(mat)], proto, stateless_rpc=req) handler.handle() @@ -263,19 +262,23 @@ class HTTPGitRequest(object): :ivar environ: the WSGI environment for the request. """ - def __init__(self, environ, start_response, dumb=False, handlers=None): + def __init__( + self, environ, start_response, dumb: bool = False, handlers=None): self.environ = environ self.dumb = dumb self.handlers = handlers self._start_response = start_response - self._cache_headers = [] - self._headers = [] + self._cache_headers = [] # type: List[Tuple[str, str]] + self._headers = [] # type: List[Tuple[str, str]] def add_header(self, name, value): """Add a header to the response.""" self._headers.append((name, value)) - def respond(self, status=HTTP_OK, content_type=None, headers=None): + def respond( + self, status: str = HTTP_OK, + content_type: Optional[str] = None, + headers: Optional[List[Tuple[str, str]]] = None): """Begin a response with the given status and other headers.""" if headers: self._headers.extend(headers) @@ -285,28 +288,28 @@ def respond(self, status=HTTP_OK, content_type=None, headers=None): return self._start_response(status, self._headers) - def not_found(self, message): + def not_found(self, message: str) -> bytes: """Begin a HTTP 404 response and return the text of a message.""" self._cache_headers = [] logger.info('Not found: %s', message) self.respond(HTTP_NOT_FOUND, 'text/plain') return message.encode('ascii') - def forbidden(self, message): + def forbidden(self, message: str) -> bytes: """Begin a HTTP 403 response and return the text of a message.""" self._cache_headers = [] logger.info('Forbidden: %s', message) self.respond(HTTP_FORBIDDEN, 'text/plain') return message.encode('ascii') - def error(self, message): + def error(self, message: str) -> bytes: """Begin a HTTP 500 response and return the text of a message.""" self._cache_headers = [] logger.error('Error: %s', message) self.respond(HTTP_ERROR, 'text/plain') return message.encode('ascii') - def nocache(self): + def nocache(self) -> None: """Set the response to never be cached by the client.""" self._cache_headers = [ ('Expires', 'Fri, 01 Jan 1980 00:00:00 GMT'), @@ -314,7 +317,7 @@ def nocache(self): ('Cache-Control', 'no-cache, max-age=0, must-revalidate'), ] - def cache_forever(self): + def cache_forever(self) -> None: """Set the response to be cached forever by the client.""" now = time.time() self._cache_headers = [ @@ -347,7 +350,9 @@ class HTTPGitApplication(object): ('POST', re.compile('/git-receive-pack$')): handle_service_request, } - def __init__(self, backend, dumb=False, handlers=None, fallback_app=None): + def __init__( + self, backend, dumb: bool = False, handlers=None, + fallback_app=None): self.backend = backend self.dumb = dumb self.handlers = dict(DEFAULT_HANDLERS) @@ -443,11 +448,8 @@ class ServerHandlerLogger(ServerHandler): """ServerHandler that uses dulwich's logger for logging exceptions.""" def log_exception(self, exc_info): - if sys.version_info < (2, 7): - logger.exception('Exception happened during processing of request') - else: - logger.exception('Exception happened during processing of request', - exc_info=exc_info) + logger.exception('Exception happened during processing of request', + exc_info=exc_info) def log_message(self, format, *args): logger.info(format, *args) diff --git a/ext_libs/giturlparse/__init__.py b/ext_libs/giturlparse/__init__.py index bcfe308f..6f25f03d 100644 --- a/ext_libs/giturlparse/__init__.py +++ b/ext_libs/giturlparse/__init__.py @@ -1,10 +1,17 @@ -# Imports +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + from .parser import parse as _parse from .result import GitUrlParsed +__author__ = 'Iacopo Spalletti' +__email__ = 'i.spalletti@nephila.it' +__version__ = '0.9.2' + def parse(url, check_domain=True): return GitUrlParsed(_parse(url, check_domain)) + def validate(url, check_domain=True): return parse(url, check_domain).valid diff --git a/ext_libs/giturlparse/parser.py b/ext_libs/giturlparse/parser.py index a41f7be2..dcef17b5 100644 --- a/ext_libs/giturlparse/parser.py +++ b/ext_libs/giturlparse/parser.py @@ -1,4 +1,6 @@ -# Imports +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + from collections import defaultdict from .platforms import PLATFORMS @@ -31,26 +33,26 @@ def parse(url, check_domain=True): # Skip if not matched if not match: - #print("[%s] URL: %s dit not match %s" % (name, url, regex.pattern)) + # print("[%s] URL: %s dit not match %s" % (name, url, regex.pattern)) continue # Skip if domain is bad domain = match.group('domain') - #print('[%s] DOMAIN = %s' % (url, domain,)) + # print('[%s] DOMAIN = %s' % (url, domain,)) if check_domain: - if platform.DOMAINS and not(domain in platform.DOMAINS): - #print("domain: %s not in %s" % (domain, platform.DOMAINS)) + if platform.DOMAINS and not (domain in platform.DOMAINS): + # print("domain: %s not in %s" % (domain, platform.DOMAINS)) continue + # add in platform defaults + parsed_info.update(platform.DEFAULTS) + # Get matches as dictionary - matches = match.groupdict() + matches = match.groupdict(default='') # Update info with matches parsed_info.update(matches) - # add in platform defaults - parsed_info.update(platform.DEFAULTS) - # Update info with platform info parsed_info.update({ 'url': url, diff --git a/ext_libs/giturlparse/platforms/__init__.py b/ext_libs/giturlparse/platforms/__init__.py index 99f6da05..f5ad612e 100644 --- a/ext_libs/giturlparse/platforms/__init__.py +++ b/ext_libs/giturlparse/platforms/__init__.py @@ -1,25 +1,22 @@ -# Imports +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +from .assembla import AssemblaPlatform from .base import BasePlatform -from .github import GitHubPlatform from .bitbucket import BitbucketPlatform from .friendcode import FriendCodePlatform -from .assembla import AssemblaPlatform -from .gogs import GogsPlatform +from .github import GitHubPlatform from .gitlab import GitLabPlatform - # Supported platforms -PLATFORMS = ( +PLATFORMS = [ # name -> Platform object ('github', GitHubPlatform()), ('bitbucket', BitbucketPlatform()), ('friendcode', FriendCodePlatform()), ('assembla', AssemblaPlatform()), - ('gogs', GogsPlatform()), ('gitlab', GitLabPlatform()), # Match url ('base', BasePlatform()), -) - -PLATFORMS_MAP = dict(PLATFORMS) +] diff --git a/ext_libs/giturlparse/platforms/assembla.py b/ext_libs/giturlparse/platforms/assembla.py index 811fd2e6..906e3e4c 100644 --- a/ext_libs/giturlparse/platforms/assembla.py +++ b/ext_libs/giturlparse/platforms/assembla.py @@ -1,4 +1,6 @@ -# Imports +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + from .base import BasePlatform diff --git a/ext_libs/giturlparse/platforms/base.py b/ext_libs/giturlparse/platforms/base.py index 28d601af..5e6b5b5b 100644 --- a/ext_libs/giturlparse/platforms/base.py +++ b/ext_libs/giturlparse/platforms/base.py @@ -1,19 +1,22 @@ -# Imports +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + import re + class BasePlatform(object): FORMATS = { 'ssh': r"%(_user)s@%(host)s:%(repo)s.git", 'http': r"http://%(host)s/%(repo)s.git", - 'https': r"http://%(host)s/%(repo)s.git", + 'https': r"https://%(host)s/%(repo)s.git", 'git': r"git://%(host)s/%(repo)s.git" } PATTERNS = { - 'ssh': r"(?P<_user>.+)s@(?P.+)s:(?P.+)s.git", - 'http': r"http://(?P.+)s/(?P.+)s.git", - 'https': r"http://(?P.+)s/(?P.+)s.git", - 'git': r"git://(?P.+)s/(?P.+)s.git" + 'ssh': r"(?P<_user>.+)@(?P.+):(?P.+).git", + 'http': r"http://(?P.+)/(?P.+).git", + 'https': r"https://(?P.+)/(?P.+).git", + 'git': r"git://(?P.+)/(?P.+).git" } # None means it matches all domains diff --git a/ext_libs/giturlparse/platforms/bitbucket.py b/ext_libs/giturlparse/platforms/bitbucket.py index 950b886c..8850511c 100644 --- a/ext_libs/giturlparse/platforms/bitbucket.py +++ b/ext_libs/giturlparse/platforms/bitbucket.py @@ -1,10 +1,13 @@ -# Imports +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + from .base import BasePlatform + class BitbucketPlatform(BasePlatform): PATTERNS = { - 'https': r'https://(?P<_user>.+)@(?P.+)/(?P.+)/(?P.+).git', - 'ssh': r'git@(?P.+):(?P.+)/(?P.+).git' + 'https': r'https://(?P<_user>.+)@(?P.+)/(?P.+)/(?P.+?)(?:\.git)?$', + 'ssh': r'git@(?P.+):(?P.+)/(?P.+?)(?:\.git)?$' } FORMATS = { 'https': r'https://%(owner)s@%(domain)s/%(owner)s/%(repo)s.git', diff --git a/ext_libs/giturlparse/platforms/friendcode.py b/ext_libs/giturlparse/platforms/friendcode.py index 2597165b..06975af8 100644 --- a/ext_libs/giturlparse/platforms/friendcode.py +++ b/ext_libs/giturlparse/platforms/friendcode.py @@ -1,4 +1,6 @@ -# Imports +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + from .base import BasePlatform diff --git a/ext_libs/giturlparse/platforms/github.py b/ext_libs/giturlparse/platforms/github.py index fa2c2bb6..cc3cc882 100644 --- a/ext_libs/giturlparse/platforms/github.py +++ b/ext_libs/giturlparse/platforms/github.py @@ -1,12 +1,14 @@ -# Imports +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + from .base import BasePlatform class GitHubPlatform(BasePlatform): PATTERNS = { - 'https': r'https://(?P.+)/(?P.+)/(?P.+).git', - 'ssh': r'git@(?P.+):(?P.+)/(?P.+).git', - 'git': r'git://(?P.+)/(?P.+)/(?P.+).git', + 'https': r'https://(?P.+)/(?P.+)/(?P.+?)(?:\.git)?$', + 'ssh': r'git@(?P.+):(?P.+)/(?P.+?)(?:\.git)?$', + 'git': r'git://(?P.+)/(?P.+)/(?P.+?)(?:\.git)?$', } FORMATS = { 'https': r'https://%(domain)s/%(owner)s/%(repo)s.git', @@ -17,4 +19,3 @@ class GitHubPlatform(BasePlatform): DEFAULTS = { '_user': 'git' } - diff --git a/ext_libs/giturlparse/platforms/gitlab.py b/ext_libs/giturlparse/platforms/gitlab.py index 4dbcee67..a2cde985 100644 --- a/ext_libs/giturlparse/platforms/gitlab.py +++ b/ext_libs/giturlparse/platforms/gitlab.py @@ -1,19 +1,24 @@ -# Imports +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + from .base import BasePlatform class GitLabPlatform(BasePlatform): PATTERNS = { - 'https': r'https://(?P.+)/(?P.+)/(?P.+).git', - 'ssh': r'git@(?P.+):(?P.+)/(?P.+).git', - 'git': r'git://(?P.+)/(?P.+)/(?P.+).git', + 'https': r'https://(?P.+)(?P:[0-9]+)?/(?P.+)/' + r'(?P.+?)(?:\.git)?$', + 'ssh': r'(ssh://)?git@(?P.+):(?P[0-9]+/)?(?P.+)/' + r'(?P.+?)(?:\.git)?$', + 'git': r'git://(?P.+):(?P[0-9]+)?/(?P.+)/' + r'(?P.+?)(?:\.git)?$', } FORMATS = { 'https': r'https://%(domain)s/%(owner)s/%(repo)s.git', - 'ssh': r'git@%(domain)s:%(owner)s/%(repo)s.git', - 'git': r'git://%(domain)s/%(owner)s/%(repo)s.git' + 'ssh': r'git@%(domain)s:%(port)s%(owner)s/%(repo)s.git', + 'git': r'git://%(domain)s%(port)s/%(owner)s/%(repo)s.git' } DEFAULTS = { - '_user': 'git' + '_user': 'git', + 'port': '' } - diff --git a/ext_libs/giturlparse/platforms/gogs.py b/ext_libs/giturlparse/platforms/gogs.py deleted file mode 100644 index 790f05c3..00000000 --- a/ext_libs/giturlparse/platforms/gogs.py +++ /dev/null @@ -1,11 +0,0 @@ -# Imports -from .base import BasePlatform - - -class GogsPlatform(BasePlatform): - PATTERNS = { - 'https': r'https://(?P.+)/(?P.+)/(?P.+).git', - } - FORMATS = { - 'https': r'https://%(domain)s/%(owner)s/%(repo)s.git', - } diff --git a/ext_libs/giturlparse/result.py b/ext_libs/giturlparse/result.py index d5d6090f..3252f4e3 100644 --- a/ext_libs/giturlparse/result.py +++ b/ext_libs/giturlparse/result.py @@ -1,6 +1,7 @@ -# Imports -from .platforms import PLATFORMS, PLATFORMS_MAP +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals +from .platforms import PLATFORMS # Possible values to extract from a Git Url REQUIRED_ATTRIBUTES = ( @@ -10,6 +11,8 @@ class GitUrlParsed(object): + platform = None + def __init__(self, parsed_info): self._parsed = parsed_info @@ -17,11 +20,16 @@ def __init__(self, parsed_info): for k, v in parsed_info.items(): setattr(self, k, v) + for name, platform in PLATFORMS: + if name == self.platform: + self._platform_obj = platform + + break + def _valid_attrs(self): - return all([ - getattr(self, attr, None) - for attr in REQUIRED_ATTRIBUTES - ]) + return all( + [getattr(self, attr, None) for attr in REQUIRED_ATTRIBUTES] + ) @property def valid(self): @@ -29,10 +37,6 @@ def valid(self): self._valid_attrs(), ]) - @property - def _platform_obj(self): - return PLATFORMS_MAP[self.platform] - ## # Alias properties ## @@ -110,10 +114,6 @@ def assembla(self): def gitlab(self): return self.platform == 'gitlab' - @property - def gogs(self): - return self.platform == 'gogs' - ## # Get data as dict ## diff --git a/ext_libs/pathvalidate/__version__.py b/ext_libs/pathvalidate/__version__.py index 064d196d..f536d775 100644 --- a/ext_libs/pathvalidate/__version__.py +++ b/ext_libs/pathvalidate/__version__.py @@ -1,6 +1,6 @@ __author__ = "Tsuyoshi Hombashi" __copyright__ = "Copyright 2016, {}".format(__author__) __license__ = "MIT License" -__version__ = "2.2.2" +__version__ = "2.3.0" __maintainer__ = __author__ __email__ = "tsuyoshi.hombashi@gmail.com" diff --git a/ext_libs/pathvalidate/_base.py b/ext_libs/pathvalidate/_base.py index a023e4ac..f7451a78 100644 --- a/ext_libs/pathvalidate/_base.py +++ b/ext_libs/pathvalidate/_base.py @@ -24,7 +24,7 @@ def platform(self) -> Platform: @property def reserved_keywords(self) -> Tuple[str, ...]: - return (".", "..") + return tuple() @property def min_len(self) -> int: diff --git a/ext_libs/pathvalidate/_filename.py b/ext_libs/pathvalidate/_filename.py index e7ccb877..f3c45a7f 100644 --- a/ext_libs/pathvalidate/_filename.py +++ b/ext_libs/pathvalidate/_filename.py @@ -181,6 +181,9 @@ def __validate_win_filename(self, unicode_filename: str) -> None: platform=Platform.WINDOWS, ) + if unicode_filename in (".", ".."): + return + if unicode_filename[-1] in (" ", "."): raise InvalidCharError( self._ERROR_MSG_TEMPLATE.format( diff --git a/ext_libs/pathvalidate/_filepath.py b/ext_libs/pathvalidate/_filepath.py index d7d0df0a..bd82b728 100644 --- a/ext_libs/pathvalidate/_filepath.py +++ b/ext_libs/pathvalidate/_filepath.py @@ -43,6 +43,7 @@ def __init__( max_len: Optional[int] = None, platform: PlatformType = None, check_reserved: bool = True, + normalize: bool = True, ) -> None: super().__init__( min_len=min_len, max_len=max_len, check_reserved=check_reserved, platform=platform, @@ -61,6 +62,7 @@ def __init__( check_reserved=check_reserved, platform=self.platform, ) + self.__normalize = normalize if self._is_universal() or self._is_windows(): self.__split_drive = ntpath.splitdrive @@ -73,9 +75,13 @@ def sanitize(self, value: PathType, replacement_text: str = "") -> PathType: self.__fpath_validator.validate_abspath(value) - unicode_file_path = preprocess(value) - drive, unicode_file_path = self.__split_drive(unicode_file_path) - sanitized_path = self._sanitize_regexp.sub(replacement_text, unicode_file_path) + unicode_filepath = preprocess(value) + + if self.__normalize: + unicode_filepath = os.path.normpath(unicode_filepath) + + drive, unicode_filepath = self.__split_drive(unicode_filepath) + sanitized_path = self._sanitize_regexp.sub(replacement_text, unicode_filepath) if self._is_windows(): path_separator = "\\" else: @@ -158,9 +164,9 @@ def validate(self, value: PathType) -> None: if not value: return - file_path = os.path.normpath(value) - unicode_file_path = preprocess(file_path) - value_len = len(unicode_file_path) + filepath = os.path.normpath(value) + unicode_filepath = preprocess(filepath) + value_len = len(unicode_filepath) if value_len > self.max_len: raise InvalidLengthError( @@ -173,18 +179,18 @@ def validate(self, value: PathType) -> None: ) ) - self._validate_reserved_keywords(unicode_file_path) - unicode_file_path = unicode_file_path.replace("\\", "/") - for entry in unicode_file_path.split("/"): + self._validate_reserved_keywords(unicode_filepath) + unicode_filepath = unicode_filepath.replace("\\", "/") + for entry in unicode_filepath.split("/"): if not entry or entry in (".", ".."): continue self.__fname_validator._validate_reserved_keywords(entry) if self._is_universal() or self._is_windows(): - self.__validate_win_file_path(unicode_file_path) + self.__validate_win_filepath(unicode_filepath) else: - self.__validate_unix_file_path(unicode_file_path) + self.__validate_unix_filepath(unicode_filepath) def validate_abspath(self, value: PathType) -> None: value = str(value) @@ -225,26 +231,26 @@ def validate_abspath(self, value: PathType) -> None: if not self._is_windows() and drive and is_nt_abs: raise err_object - def __validate_unix_file_path(self, unicode_file_path: str) -> None: - match = _RE_INVALID_PATH.findall(unicode_file_path) + def __validate_unix_filepath(self, unicode_filepath: str) -> None: + match = _RE_INVALID_PATH.findall(unicode_filepath) if match: raise InvalidCharError( self._ERROR_MSG_TEMPLATE.format( - invalid=findall_to_str(match), value=repr(unicode_file_path) + invalid=findall_to_str(match), value=repr(unicode_filepath) ) ) - def __validate_win_file_path(self, unicode_file_path: str) -> None: - match = _RE_INVALID_WIN_PATH.findall(unicode_file_path) + def __validate_win_filepath(self, unicode_filepath: str) -> None: + match = _RE_INVALID_WIN_PATH.findall(unicode_filepath) if match: raise InvalidCharError( self._ERROR_MSG_TEMPLATE.format( - invalid=findall_to_str(match), value=repr(unicode_file_path) + invalid=findall_to_str(match), value=repr(unicode_filepath) ), platform=Platform.WINDOWS, ) - _drive, value = self.__split_drive(unicode_file_path) + _drive, value = self.__split_drive(unicode_filepath) if value: match_reserved = self._RE_NTFS_RESERVED.search(value) if match_reserved: @@ -334,6 +340,7 @@ def sanitize_filepath( platform: Optional[str] = None, max_len: Optional[int] = None, check_reserved: bool = True, + normalize: bool = True, ) -> PathType: """Make a valid file path from a string. @@ -363,7 +370,7 @@ def sanitize_filepath( max_len: Maximum length of the ``file_path`` length. Truncate the name if the ``file_path`` length exceedd this value. If the value is |None|, - automatically determined by the ``platform``: + ``max_len`` will automatically determined by the ``platform``: - ``Linux``: 4096 - ``macOS``: 1024 @@ -371,6 +378,8 @@ def sanitize_filepath( - ``universal``: 260 check_reserved: If |True|, sanitize reserved names of the ``platform``. + normalize: + If |True|, normalize the the file path. Returns: Same type as the argument (str or PathLike object): @@ -385,7 +394,7 @@ def sanitize_filepath( """ return FilePathSanitizer( - platform=platform, max_len=max_len, check_reserved=check_reserved + platform=platform, max_len=max_len, check_reserved=check_reserved, normalize=normalize ).sanitize(file_path, replacement_text) diff --git a/ext_libs/pathvalidate/_symbol.py b/ext_libs/pathvalidate/_symbol.py index 573b1c2d..663538ae 100644 --- a/ext_libs/pathvalidate/_symbol.py +++ b/ext_libs/pathvalidate/_symbol.py @@ -17,12 +17,14 @@ def validate_unprintable(text: str) -> None: + # deprecated match_list = __RE_UNPRINTABLE.findall(preprocess(text)) if match_list: raise InvalidCharError("unprintable character found: {}".format(match_list)) def replace_unprintable(text: str, replacement_text: str = "") -> str: + # deprecated try: return __RE_UNPRINTABLE.sub(replacement_text, preprocess(text)) except (TypeError, AttributeError): diff --git a/requirements/embedded.txt b/requirements/embedded.txt new file mode 100644 index 00000000..7b8bbfe2 --- /dev/null +++ b/requirements/embedded.txt @@ -0,0 +1,6 @@ +# Embedded requirements +# ----------------------- + +dulwich>=0.20,<0.21 +giturlparse>=0.9,<0.10 +pathvalidate>=2.3,<2.4