Embedded dependencies workflow

.gitignore

@@ -20,6 +20,7 @@ lib64/
 parts/
 sdist/
 var/
+*.dist-info/
 *.egg-info/
 .installed.cfg
 *.egg
@@ -33,3 +34,21 @@ nosetests.xml
 coverage.xml
 *,cover
 
+# Sphinx documentation
+docs/_build/
+
+#############
+## Custom
+#############
+
+# virtualenvs
+venv/
+ENV/
+.venv*
+.venv/*
+
+# embedded libs
+ext_libs/bin
+ext_libs/docs
+ext_libs/*/contrib/
+ext_libs/*/tests/

docs/development/update_dependencies.md

@@ -0,0 +1,24 @@
+# Dependencies upgrade workflow
+
+This plugin is based on external dependencies:
+
+- dulwich
+- giturlparse
+- pathvalidate
+
+Because it's still hard to instal Python 3rd party packages from an index (for example <https://pypi.org>), especially on Windows or Mac systems (or even on Linux if we want to do it properly in a virtual environment), those required packages are stored into the `ext_libs` folder.
+
+## Upgrade workflow
+
+Manage versions in the `requirements/embedded.txt` file, then:
+
+```bash
+python -m pip install --no-deps -U -r requirements/embedded.txt -t ext_libs
+```
+
+Note: even if `dulwich` depends on `certifi` and `urllib3`, we specifally install them since they are already included with QGIS.
+
+## Related links
+
+- <https://gis.stackexchange.com/questions/141320/installing-3rd-party-python-libraries-for-qgis-on-windows>
+- <https://github.com/QGIS-Contribution/QGIS-ResourceSharing/issues/112>

ext_libs/dulwich/__init__.py

@@ -22,4 +22,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 19, 15)
+__version__ = (0, 20, 6)

ext_libs/dulwich/_diff_tree.c

@@ -26,25 +26,6 @@
 typedef unsigned short mode_t;
 #endif
 
-#if PY_MAJOR_VERSION < 3
-typedef long Py_hash_t;
-#endif
-
-#if PY_MAJOR_VERSION >= 3
-#define PyInt_FromLong PyLong_FromLong
-#define PyInt_AsLong PyLong_AsLong
-#define PyInt_AS_LONG PyLong_AS_LONG
-#define PyString_AS_STRING PyBytes_AS_STRING
-#define PyString_AsStringAndSize PyBytes_AsStringAndSize
-#define PyString_Check PyBytes_Check
-#define PyString_CheckExact PyBytes_CheckExact
-#define PyString_FromStringAndSize PyBytes_FromStringAndSize
-#define PyString_FromString PyBytes_FromString
-#define PyString_GET_SIZE PyBytes_GET_SIZE
-#define PyString_Size PyBytes_Size
-#define _PyString_Join _PyBytes_Join
-#endif
-
 static PyObject *tree_entry_cls = NULL, *null_entry = NULL,
 	*defaultdict_cls = NULL, *int_cls = NULL;
 static int block_size;
@@ -118,7 +99,7 @@ static PyObject **tree_entries(char *path, Py_ssize_t path_len, PyObject *tree,
 		if (!sha)
 			goto error;
 		name = PyTuple_GET_ITEM(old_entry, 0);
-		name_len = PyString_Size(name);
+		name_len = PyBytes_Size(name);
 		if (PyErr_Occurred())
 			goto error;
 
@@ -133,18 +114,13 @@ static PyObject **tree_entries(char *path, Py_ssize_t path_len, PyObject *tree,
 		if (path_len) {
 			memcpy(new_path, path, path_len);
 			new_path[path_len] = '/';
-			memcpy(new_path + path_len + 1, PyString_AS_STRING(name), name_len);
+			memcpy(new_path + path_len + 1, PyBytes_AS_STRING(name), name_len);
 		} else {
-			memcpy(new_path, PyString_AS_STRING(name), name_len);
+			memcpy(new_path, PyBytes_AS_STRING(name), name_len);
 		}
 
-#if PY_MAJOR_VERSION >= 3
 		result[i] = PyObject_CallFunction(tree_entry_cls, "y#OO", new_path,
 			new_path_len, PyTuple_GET_ITEM(old_entry, 1), sha);
-#else
-		result[i] = PyObject_CallFunction(tree_entry_cls, "s#OO", new_path,
-			new_path_len, PyTuple_GET_ITEM(old_entry, 1), sha);
-#endif
 		PyMem_Free(new_path);
 		if (!result[i]) {
 			goto error;
@@ -172,7 +148,7 @@ static int entry_path_cmp(PyObject *entry1, PyObject *entry2)
 	if (!path1)
 		goto done;
 
-	if (!PyString_Check(path1)) {
+	if (!PyBytes_Check(path1)) {
 		PyErr_SetString(PyExc_TypeError, "path is not a (byte)string");
 		goto done;
 	}
@@ -181,12 +157,12 @@ static int entry_path_cmp(PyObject *entry1, PyObject *entry2)
 	if (!path2)
 		goto done;
 
-	if (!PyString_Check(path2)) {
+	if (!PyBytes_Check(path2)) {
 		PyErr_SetString(PyExc_TypeError, "path is not a (byte)string");
 		goto done;
 	}
 
-	result = strcmp(PyString_AS_STRING(path1), PyString_AS_STRING(path2));
+	result = strcmp(PyBytes_AS_STRING(path1), PyBytes_AS_STRING(path2));
 
 done:
 	Py_XDECREF(path1);
@@ -202,11 +178,7 @@ static PyObject *py_merge_entries(PyObject *self, PyObject *args)
 	char *path_str;
 	int cmp;
 
-#if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTuple(args, "y#OO", &path_str, &path_len, &tree1, &tree2))
-#else
-	if (!PyArg_ParseTuple(args, "s#OO", &path_str, &path_len, &tree1, &tree2))
-#endif
 		return NULL;
 
 	entries1 = tree_entries(path_str, path_len, tree1, &n1);
@@ -270,6 +242,11 @@ static PyObject *py_merge_entries(PyObject *self, PyObject *args)
 	return result;
 }
 
+/* Not all environments define S_ISDIR */
+#if !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR)
+#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
+#endif
+
 static PyObject *py_is_tree(PyObject *self, PyObject *args)
 {
 	PyObject *entry, *mode, *result;
@@ -286,7 +263,7 @@ static PyObject *py_is_tree(PyObject *self, PyObject *args)
 		result = Py_False;
 		Py_INCREF(result);
 	} else {
-		lmode = PyInt_AsLong(mode);
+		lmode = PyLong_AsLong(mode);
 		if (lmode == -1 && PyErr_Occurred()) {
 			Py_DECREF(mode);
 			return NULL;
@@ -305,21 +282,21 @@ static Py_hash_t add_hash(PyObject *get, PyObject *set, char *str, int n)
 
 	/* It would be nice to hash without copying str into a PyString, but that
 	 * isn't exposed by the API. */
-	str_obj = PyString_FromStringAndSize(str, n);
+	str_obj = PyBytes_FromStringAndSize(str, n);
 	if (!str_obj)
 		goto error;
 	hash = PyObject_Hash(str_obj);
 	if (hash == -1)
 		goto error;
-	hash_obj = PyInt_FromLong(hash);
+	hash_obj = PyLong_FromLong(hash);
 	if (!hash_obj)
 		goto error;
 
 	value = PyObject_CallFunctionObjArgs(get, hash_obj, NULL);
 	if (!value)
 		goto error;
 	set_value = PyObject_CallFunction(set, "(Ol)", hash_obj,
-		PyInt_AS_LONG(value) + n);
+		PyLong_AS_LONG(value) + n);
 	if (!set_value)
 		goto error;
 
@@ -372,11 +349,11 @@ static PyObject *py_count_blocks(PyObject *self, PyObject *args)
 
 	for (i = 0; i < num_chunks; i++) {
 		chunk = PyList_GET_ITEM(chunks, i);
-		if (!PyString_Check(chunk)) {
+		if (!PyBytes_Check(chunk)) {
 			PyErr_SetString(PyExc_TypeError, "chunk is not a string");
 			goto error;
 		}
-		if (PyString_AsStringAndSize(chunk, &chunk_str, &chunk_len) == -1)
+		if (PyBytes_AsStringAndSize(chunk, &chunk_str, &chunk_len) == -1)
 			goto error;
 
 		for (j = 0; j < chunk_len; j++) {
@@ -420,7 +397,6 @@ moduleinit(void)
 	PyObject *m, *objects_mod = NULL, *diff_tree_mod = NULL;
 	PyObject *block_size_obj = NULL;
 
-#if PY_MAJOR_VERSION >= 3
 	static struct PyModuleDef moduledef = {
 		PyModuleDef_HEAD_INIT,
 		"_diff_tree",         /* m_name */
@@ -433,9 +409,6 @@ moduleinit(void)
 		NULL,                 /* m_free */
 	};
 	m = PyModule_Create(&moduledef);
-#else
-	m = Py_InitModule("_diff_tree", py_diff_tree_methods);
-#endif
 	if (!m)
 		goto error;
 
@@ -459,7 +432,7 @@ moduleinit(void)
 	block_size_obj = PyObject_GetAttrString(diff_tree_mod, "_BLOCK_SIZE");
 	if (!block_size_obj)
 		goto error;
-	block_size = (int)PyInt_AsLong(block_size_obj);
+	block_size = (int)PyLong_AsLong(block_size_obj);
 
 	if (PyErr_Occurred())
 		goto error;
@@ -490,16 +463,8 @@ moduleinit(void)
 	return NULL;
 }
 
-#if PY_MAJOR_VERSION >= 3
 PyMODINIT_FUNC
 PyInit__diff_tree(void)
 {
 	return moduleinit();
 }
-#else
-PyMODINIT_FUNC
-init_diff_tree(void)
-{
-	moduleinit();
-}
-#endif

ext_libs/dulwich/_objects.c

@@ -23,15 +23,6 @@
 #include <stdlib.h>
 #include <sys/stat.h>
 
-#if PY_MAJOR_VERSION >= 3
-#define PyInt_Check(obj) 0
-#define PyInt_CheckExact(obj) 0
-#define PyInt_AsLong PyLong_AsLong
-#define PyString_AS_STRING PyBytes_AS_STRING
-#define PyString_Check PyBytes_Check
-#define PyString_FromStringAndSize PyBytes_FromStringAndSize
-#endif
-
 #if defined(__MINGW32_VERSION) || defined(__APPLE__)
 size_t rep_strnlen(char *text, size_t maxlen);
 size_t rep_strnlen(char *text, size_t maxlen)
@@ -56,7 +47,7 @@ static PyObject *sha_to_pyhex(const unsigned char *sha)
 		hexsha[i*2+1] = bytehex(sha[i] & 0x0F);
 	}
 
-	return PyString_FromStringAndSize(hexsha, 40);
+	return PyBytes_FromStringAndSize(hexsha, 40);
 }
 
 static PyObject *py_parse_tree(PyObject *self, PyObject *args, PyObject *kw)
@@ -67,13 +58,8 @@ static PyObject *py_parse_tree(PyObject *self, PyObject *args, PyObject *kw)
 	PyObject *ret, *item, *name, *sha, *py_strict = NULL;
 	static char *kwlist[] = {"text", "strict", NULL};
 
-#if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTupleAndKeywords(args, kw, "y#|O", kwlist,
 	                                 &text, &len, &py_strict))
-#else
-	if (!PyArg_ParseTupleAndKeywords(args, kw, "s#|O", kwlist,
-	                                 &text, &len, &py_strict))
-#endif
 		return NULL;
 	strict = py_strict ?  PyObject_IsTrue(py_strict) : 0;
 	/* TODO: currently this returns a list; if memory usage is a concern,
@@ -100,7 +86,7 @@ static PyObject *py_parse_tree(PyObject *self, PyObject *args, PyObject *kw)
 		}
 		text++;
 		namelen = strnlen(text, len - (text - start));
-		name = PyString_FromStringAndSize(text, namelen);
+		name = PyBytes_FromStringAndSize(text, namelen);
 		if (name == NULL) {
 			Py_DECREF(ret);
 			return NULL;
@@ -141,6 +127,11 @@ struct tree_item {
 	PyObject *tuple;
 };
 
+/* Not all environments define S_ISDIR */
+#if !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR)
+#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
+#endif
+
 int cmp_tree_item(const void *_a, const void *_b)
 {
 	const struct tree_item *a = _a, *b = _b;
@@ -202,7 +193,7 @@ static PyObject *py_sorted_tree_items(PyObject *self, PyObject *args)
 	}
 
 	while (PyDict_Next(entries, &pos, &key, &value)) {
-		if (!PyString_Check(key)) {
+		if (!PyBytes_Check(key)) {
 			PyErr_SetString(PyExc_TypeError, "Name is not a string");
 			goto error;
 		}
@@ -213,18 +204,18 @@ static PyObject *py_sorted_tree_items(PyObject *self, PyObject *args)
 		}
 
 		py_mode = PyTuple_GET_ITEM(value, 0);
-		if (!PyInt_Check(py_mode) && !PyLong_Check(py_mode)) {
+		if (!PyLong_Check(py_mode)) {
 			PyErr_SetString(PyExc_TypeError, "Mode is not an integral type");
 			goto error;
 		}
 
 		py_sha = PyTuple_GET_ITEM(value, 1);
-		if (!PyString_Check(py_sha)) {
+		if (!PyBytes_Check(py_sha)) {
 			PyErr_SetString(PyExc_TypeError, "SHA is not a string");
 			goto error;
 		}
-		qsort_entries[n].name = PyString_AS_STRING(key);
-		qsort_entries[n].mode = PyInt_AsLong(py_mode);
+		qsort_entries[n].name = PyBytes_AS_STRING(key);
+		qsort_entries[n].mode = PyLong_AsLong(py_mode);
 
 		qsort_entries[n].tuple = PyObject_CallFunctionObjArgs(
 		                tree_entry_cls, key, py_mode, py_sha, NULL);
@@ -267,7 +258,6 @@ moduleinit(void)
 {
 	PyObject *m, *objects_mod, *errors_mod;
 
-#if PY_MAJOR_VERSION >= 3
 	static struct PyModuleDef moduledef = {
 		PyModuleDef_HEAD_INIT,
 		"_objects",         /* m_name */
@@ -280,9 +270,6 @@ moduleinit(void)
 		NULL,               /* m_free */
 	};
 	m = PyModule_Create(&moduledef);
-#else
-	m = Py_InitModule3("_objects", py_objects_methods, NULL);
-#endif
 	if (m == NULL) {
 		return NULL;
 	}
@@ -315,16 +302,8 @@ moduleinit(void)
 	return m;
 }
 
-#if PY_MAJOR_VERSION >= 3
 PyMODINIT_FUNC
 PyInit__objects(void)
 {
 	return moduleinit();
 }
-#else
-PyMODINIT_FUNC
-init_objects(void)
-{
-	moduleinit();
-}
-#endif