From b652a3eb6de7caaada334972a2c7a8e70f9db376 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Mon, 2 Dec 2019 20:10:31 -0800 Subject: [PATCH 1/7] adding repoproviders documentation --- binderhub/repoproviders.py | 15 +++++---- doc/api.rst | 31 +++++------------ doc/developer/index.rst | 1 + doc/developer/repoproviders.rst | 60 +++++++++++++++++++++++++++++++++ doc/reference/repoproviders.rst | 38 +++++++++++++++++++++ 5 files changed, 115 insertions(+), 30 deletions(-) create mode 100644 doc/developer/repoproviders.rst diff --git a/binderhub/repoproviders.py b/binderhub/repoproviders.py index 3b0ae53a9..b6417c71c 100644 --- a/binderhub/repoproviders.py +++ b/binderhub/repoproviders.py @@ -1,11 +1,11 @@ """ -Classes for Repo providers +Classes for Repo providers. Subclass the base class, ``RepoProvider``, to support different version control services and providers. -Note: When adding a new repo provider, add it to the allowed values for - repo providers in event-schemas/launch.json. +.. note:: When adding a new repo provider, add it to the allowed values for + repo providers in event-schemas/launch.json. """ from datetime import timedelta import json @@ -339,7 +339,7 @@ def get_resolved_ref(self): .format(self.unresolved_ref)) resolved_ref = result.stdout.split(None, 1)[0] self.sha1_validate(resolved_ref) - self.resolved_ref = resolved_ref + self.resolved_ref = resolved_ref else: # The ref already was a valid SHA hash self.resolved_ref = self.unresolved_ref @@ -499,7 +499,7 @@ class GitHubRepoProvider(RepoProvider): api_base_path = Unicode('https://api.{hostname}', config=True, help="""The base path of the GitHub API - + Only necessary if not github.com, e.g. GitHub Enterprise. @@ -652,7 +652,7 @@ def github_api_request(self, api_url, etag=None): log("GitHub rate limit remaining {remaining}/{limit}. Reset in {delta}.".format( remaining=remaining, limit=rate_limit, delta=delta, )) - + return resp @gen.coroutine @@ -661,7 +661,7 @@ def get_resolved_ref(self): return self.resolved_ref api_url = "{api_base_path}/repos/{user}/{repo}/commits/{ref}".format( - api_base_path=self.api_base_path.format(hostname=self.hostname), + api_base_path=self.api_base_path.format(hostname=self.hostname), user=self.user, repo=self.repo, ref=self.unresolved_ref ) self.log.debug("Fetching %s", api_url) @@ -720,6 +720,7 @@ class GistRepoProvider(GitHubRepoProvider): The ref is optional, valid values are - a full sha1 of a ref in the history - master + If master or no ref is specified the latest revision will be used. """ diff --git a/doc/api.rst b/doc/api.rst index 6520699fa..bf37eac87 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -12,31 +12,15 @@ There's one API endpoint, which is: Even though it says **build** it actually performs **launch**. -**provider_prefix** identifies the provider. -**spec** defines the source of the computing environment to be built and -served using the given provider. -See :ref:`providers-section` for supported inputs. +* **provider_prefix** identifies the provider. +* **spec** defines the source of the computing environment to be built and + served using the given provider. -.. _providers-section: +.. note:: The **provider_prefix** can be any of the supported + repository providers in BinderHub, see the + :ref:`providers-section` section for supported inputs. -Providers ---------- - -Currently supported providers, their prefixes and specs are: - -+------------+--------------------+-------------------------------------------------------------+----------------------------+ -| Provider | provider_prefix | spec | notes | -+============+====================+=============================================================+============================+ -| GitHub | ``gh`` | ``//`` | | -+------------+--------------------+-------------------------------------------------------------+----------------------------+ -| Git | ``git`` | ``/`` | arbitrary HTTP git repos | -+------------+--------------------+-------------------------------------------------------------+----------------------------+ -| GitLab | ``gl`` | ``/`` | | -+------------+--------------------+-------------------------------------------------------------+----------------------------+ -| Gist | ``gist`` | ``/`` | | -+------------+--------------------+-------------------------------------------------------------+----------------------------+ - -Next, construct an appropriate URL and send a request. +To use this endpoint, construct an appropriate URL and send a request. You'll get back an `Event Stream `__. @@ -56,6 +40,7 @@ When the request is received, the following happens: 4. If the build succeeds, we contact the JupyterHub API and start launching the server. + Events ------ diff --git a/doc/developer/index.rst b/doc/developer/index.rst index 21e661778..ef81e672e 100644 --- a/doc/developer/index.rst +++ b/doc/developer/index.rst @@ -10,4 +10,5 @@ A more detailed overview of the BinderHub design, architecture, and functionalit ../overview ../eventlogging ../api + repoproviders ../reference/ref-index.rst diff --git a/doc/developer/repoproviders.rst b/doc/developer/repoproviders.rst new file mode 100644 index 000000000..4d35469df --- /dev/null +++ b/doc/developer/repoproviders.rst @@ -0,0 +1,60 @@ +.. _providers-section: + +==================== +Repository Providers +==================== + +Repository Providers (or RepoProviders) are +locations where git repositories are stored (e.g., +GitHub). BinderHub supports a number of providers out of the +box, and can be extended to support new providers. For a complete +listing of the provider classes, see :ref:`api-repoproviders`. + +Supported repoproviders +======================= + +Currently supported providers, their prefixes and specs are: + ++------------+--------------------+-------------------------------------------------------------+----------------------------+ +| Provider | provider_prefix | spec | notes | ++============+====================+=============================================================+============================+ +| GitHub | ``gh`` | ``//`` | | ++------------+--------------------+-------------------------------------------------------------+----------------------------+ +| GitLab | ``gl`` | ``//`` | | ++------------+--------------------+-------------------------------------------------------------+----------------------------+ +| Gist | ``gist`` | ``/`` | | ++------------+--------------------+-------------------------------------------------------------+----------------------------+ +| Zenodo | ``zenodo`` | ```` | | ++------------+--------------------+-------------------------------------------------------------+----------------------------+ +| Figshare | ``figshare`` | ```` | | ++------------+--------------------+-------------------------------------------------------------+----------------------------+ +| Git | ``git`` | ``/`` | arbitrary HTTP git repos | ++------------+--------------------+-------------------------------------------------------------+----------------------------+ + +Adding a new repository provider +================================ + +It is possible to add new repository providers to BinderHub, allowing +a BinderHub deployment to fetch git repositories from new locations +on the web. Doing so involves defining your own RepoProvider sub-class +and modifying a set of methods/attributes to interface with the online +provider to which you are providing access. + +In order to extend the supported repository providers, +follow these instructions. We'll provide example links for each step to a +recent `BinderHub pull-request `_ +that implements the ``DataverseProvider`` class. + +1. Review the `repoprovider module `_. + This shows a number of example repository providers. +2. Create a new class that sub-classes the ``RepoProvider`` class. + Define your own methods for actions that are repository provider-specific. + For example, `here is the DataverseProvider class `_. +3. Add this class to the `list of default RepoProviders in BinderHub `_. +4. Add the new provider prefix `to the BinderHub UI `_ + and `the index javascript `_ + and make the appropriate changes to the index page based on the URL + specification for this repository provider. +5. Add `a test for your repoprovider `_ + to ensure that it properly resolves and fetches a repository URL. +6. Document your new repository provider on the :ref:`providers-section` page. diff --git a/doc/reference/repoproviders.rst b/doc/reference/repoproviders.rst index 8f12bc04d..f058aa88a 100644 --- a/doc/reference/repoproviders.rst +++ b/doc/reference/repoproviders.rst @@ -1,3 +1,5 @@ +.. _api-repoproviders: + repoproviders ============= @@ -14,8 +16,44 @@ Module: :mod:`binderhub.repoproviders` .. autoconfigurable:: RepoProvider :members: + :class:`GitHubRepoProvider` --------------------------- .. autoconfigurable:: GitHubRepoProvider :members: + + +:class:`GitLabRepoProvider` +--------------------------- + +.. autoconfigurable:: GitLabRepoProvider + :members: + + +:class:`GistRepoProvider` +--------------------------- + +.. autoconfigurable:: GistRepoProvider + :members: + + +:class:`ZenodoProvider` +--------------------------- + +.. autoconfigurable:: ZenodoProvider + :members: + + +:class:`FigshareProvider` +--------------------------- + +.. autoconfigurable:: FigshareProvider + :members: + + +:class:`GitRepoProvider` +--------------------------- + +.. autoconfigurable:: GitRepoProvider + :members: From 0f08373165d5b4b662ab477305da5b11db252cf6 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Sun, 1 Dec 2019 17:56:47 -0800 Subject: [PATCH 2/7] Update doc/developer/repoproviders.rst Co-Authored-By: Tim Head --- doc/developer/repoproviders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/developer/repoproviders.rst b/doc/developer/repoproviders.rst index 4d35469df..37a9dd4fa 100644 --- a/doc/developer/repoproviders.rst +++ b/doc/developer/repoproviders.rst @@ -5,7 +5,7 @@ Repository Providers ==================== Repository Providers (or RepoProviders) are -locations where git repositories are stored (e.g., +locations where repositories are stored (e.g., GitHub). BinderHub supports a number of providers out of the box, and can be extended to support new providers. For a complete listing of the provider classes, see :ref:`api-repoproviders`. From fed013297b61a66a57a8a282c6a9c4e9e22d6d62 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Sun, 1 Dec 2019 17:57:07 -0800 Subject: [PATCH 3/7] Update doc/developer/repoproviders.rst Co-Authored-By: Tim Head --- doc/developer/repoproviders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/developer/repoproviders.rst b/doc/developer/repoproviders.rst index 37a9dd4fa..4945efb57 100644 --- a/doc/developer/repoproviders.rst +++ b/doc/developer/repoproviders.rst @@ -35,7 +35,7 @@ Adding a new repository provider ================================ It is possible to add new repository providers to BinderHub, allowing -a BinderHub deployment to fetch git repositories from new locations +a BinderHub deployment to fetch repositories from new locations on the web. Doing so involves defining your own RepoProvider sub-class and modifying a set of methods/attributes to interface with the online provider to which you are providing access. From 7a287713c9d9d5ae7fbb1c2bf4a130be39a657cd Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Sun, 1 Dec 2019 18:05:10 -0800 Subject: [PATCH 4/7] note about content providers --- doc/developer/repoproviders.rst | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/developer/repoproviders.rst b/doc/developer/repoproviders.rst index 4945efb57..21439dc28 100644 --- a/doc/developer/repoproviders.rst +++ b/doc/developer/repoproviders.rst @@ -38,23 +38,26 @@ It is possible to add new repository providers to BinderHub, allowing a BinderHub deployment to fetch repositories from new locations on the web. Doing so involves defining your own RepoProvider sub-class and modifying a set of methods/attributes to interface with the online -provider to which you are providing access. +provider to which you are providing access. It also often involves +`building a new repo2docker content provider `_. In order to extend the supported repository providers, follow these instructions. We'll provide example links for each step to a recent `BinderHub pull-request `_ that implements the ``DataverseProvider`` class. -1. Review the `repoprovider module `_. +#. Review the `repoprovider module `_. This shows a number of example repository providers. -2. Create a new class that sub-classes the ``RepoProvider`` class. +#. Check whether repo2docker has a `ContentProvider class `_ + that will work with your repository provider. If not, then you'll need to create one first. +#. Create a new class that sub-classes the ``RepoProvider`` class. Define your own methods for actions that are repository provider-specific. For example, `here is the DataverseProvider class `_. -3. Add this class to the `list of default RepoProviders in BinderHub `_. -4. Add the new provider prefix `to the BinderHub UI `_ +#. Add this class to the `list of default RepoProviders in BinderHub `_. +#. Add the new provider prefix `to the BinderHub UI `_ and `the index javascript `_ and make the appropriate changes to the index page based on the URL specification for this repository provider. -5. Add `a test for your repoprovider `_ +#. Add `a test for your repoprovider `_ to ensure that it properly resolves and fetches a repository URL. -6. Document your new repository provider on the :ref:`providers-section` page. +#. Document your new repository provider on the :ref:`providers-section` page. From 35ec76a4dc008222e3084c144d9b4a8ef9ab0b0d Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Wed, 4 Dec 2019 13:07:49 -0800 Subject: [PATCH 5/7] Update repoproviders.rst --- doc/reference/repoproviders.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/reference/repoproviders.rst b/doc/reference/repoproviders.rst index f058aa88a..3e7fd557e 100644 --- a/doc/reference/repoproviders.rst +++ b/doc/reference/repoproviders.rst @@ -13,6 +13,8 @@ Module: :mod:`binderhub.repoproviders` :class:`RepoProvider` --------------------- +A generic repository provider class. + .. autoconfigurable:: RepoProvider :members: @@ -20,6 +22,8 @@ Module: :mod:`binderhub.repoproviders` :class:`GitHubRepoProvider` --------------------------- +`GitHub `_ is a website for hosting and sharing git repositories. + .. autoconfigurable:: GitHubRepoProvider :members: @@ -27,6 +31,9 @@ Module: :mod:`binderhub.repoproviders` :class:`GitLabRepoProvider` --------------------------- +`GitLab `_ offers hosted as well as self-hosted git +repositories. + .. autoconfigurable:: GitLabRepoProvider :members: @@ -34,6 +41,9 @@ Module: :mod:`binderhub.repoproviders` :class:`GistRepoProvider` --------------------------- +`Gists `_ are small collections of files stored on GitHub. They +behave like lightweight repositories. + .. autoconfigurable:: GistRepoProvider :members: @@ -41,6 +51,9 @@ Module: :mod:`binderhub.repoproviders` :class:`ZenodoProvider` --------------------------- +`Zenodo `_ is a non-profit provider of scholarly artifacts +(such as code repositories) run in partnership with CERN. + .. autoconfigurable:: ZenodoProvider :members: @@ -48,6 +61,9 @@ Module: :mod:`binderhub.repoproviders` :class:`FigshareProvider` --------------------------- +`FigShare `_ is company that offers hosting for scholarly +artifacts (such as code repositories). + .. autoconfigurable:: FigshareProvider :members: @@ -55,5 +71,7 @@ Module: :mod:`binderhub.repoproviders` :class:`GitRepoProvider` --------------------------- +A generic repository provider for URLs that point directly to a git repository. + .. autoconfigurable:: GitRepoProvider :members: From 99dd5e6df86a261b24bd863e21fc0774884d212e Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Wed, 4 Dec 2019 13:08:52 -0800 Subject: [PATCH 6/7] Update repoproviders.rst --- doc/developer/repoproviders.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/developer/repoproviders.rst b/doc/developer/repoproviders.rst index 21439dc28..92ed9b250 100644 --- a/doc/developer/repoproviders.rst +++ b/doc/developer/repoproviders.rst @@ -60,4 +60,5 @@ that implements the ``DataverseProvider`` class. specification for this repository provider. #. Add `a test for your repoprovider `_ to ensure that it properly resolves and fetches a repository URL. -#. Document your new repository provider on the :ref:`providers-section` page. +#. Document your new repository provider on the :ref:`providers-section` page as well + as the :ref:`api-repoproviders` page. From 0316406193059060ebb429b3c342b76b14148882 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Wed, 4 Dec 2019 13:35:15 -0800 Subject: [PATCH 7/7] moving repoprovider information to the table --- doc/developer/repoproviders.rst | 33 ++++++++++++++++++--------------- doc/reference/repoproviders.rst | 18 ------------------ 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/doc/developer/repoproviders.rst b/doc/developer/repoproviders.rst index 92ed9b250..0295560ac 100644 --- a/doc/developer/repoproviders.rst +++ b/doc/developer/repoproviders.rst @@ -15,21 +15,24 @@ Supported repoproviders Currently supported providers, their prefixes and specs are: -+------------+--------------------+-------------------------------------------------------------+----------------------------+ -| Provider | provider_prefix | spec | notes | -+============+====================+=============================================================+============================+ -| GitHub | ``gh`` | ``//`` | | -+------------+--------------------+-------------------------------------------------------------+----------------------------+ -| GitLab | ``gl`` | ``//`` | | -+------------+--------------------+-------------------------------------------------------------+----------------------------+ -| Gist | ``gist`` | ``/`` | | -+------------+--------------------+-------------------------------------------------------------+----------------------------+ -| Zenodo | ``zenodo`` | ```` | | -+------------+--------------------+-------------------------------------------------------------+----------------------------+ -| Figshare | ``figshare`` | ```` | | -+------------+--------------------+-------------------------------------------------------------+----------------------------+ -| Git | ``git`` | ``/`` | arbitrary HTTP git repos | -+------------+--------------------+-------------------------------------------------------------+----------------------------+ +.. table:: + :widths: 5 5 55 35 + + +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+ + | Provider | provider_prefix | spec | notes | + +============+====================+=============================================================+===========================================================================================================================================+ + | GitHub | ``gh`` | ``//`` | `GitHub `_ is a website for hosting and sharing git repositories. | + +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+ + | GitLab | ``gl`` | ``//`` | `GitLab `_ offers hosted as well as self-hosted git repositories. | + +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+ + | Gist | ``gist`` | ``/`` | `Gists `_ are small collections of files stored on GitHub. They behave like lightweight repositories. | + +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+ + | Zenodo | ``zenodo`` | ```` | `Zenodo `_ is a non-profit provider of scholarly artifacts (such as code repositories) run in partnership with CERN. | + +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+ + | Figshare | ``figshare`` | ```` | `FigShare `_ is company that offers hosting for scholarly artifacts (such as code repositories). | + +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+ + | Git | ``git`` | ``/`` | A generic repository provider for URLs that point directly to a git repository. | + +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+ Adding a new repository provider ================================ diff --git a/doc/reference/repoproviders.rst b/doc/reference/repoproviders.rst index 3e7fd557e..f058aa88a 100644 --- a/doc/reference/repoproviders.rst +++ b/doc/reference/repoproviders.rst @@ -13,8 +13,6 @@ Module: :mod:`binderhub.repoproviders` :class:`RepoProvider` --------------------- -A generic repository provider class. - .. autoconfigurable:: RepoProvider :members: @@ -22,8 +20,6 @@ A generic repository provider class. :class:`GitHubRepoProvider` --------------------------- -`GitHub `_ is a website for hosting and sharing git repositories. - .. autoconfigurable:: GitHubRepoProvider :members: @@ -31,9 +27,6 @@ A generic repository provider class. :class:`GitLabRepoProvider` --------------------------- -`GitLab `_ offers hosted as well as self-hosted git -repositories. - .. autoconfigurable:: GitLabRepoProvider :members: @@ -41,9 +34,6 @@ repositories. :class:`GistRepoProvider` --------------------------- -`Gists `_ are small collections of files stored on GitHub. They -behave like lightweight repositories. - .. autoconfigurable:: GistRepoProvider :members: @@ -51,9 +41,6 @@ behave like lightweight repositories. :class:`ZenodoProvider` --------------------------- -`Zenodo `_ is a non-profit provider of scholarly artifacts -(such as code repositories) run in partnership with CERN. - .. autoconfigurable:: ZenodoProvider :members: @@ -61,9 +48,6 @@ behave like lightweight repositories. :class:`FigshareProvider` --------------------------- -`FigShare `_ is company that offers hosting for scholarly -artifacts (such as code repositories). - .. autoconfigurable:: FigshareProvider :members: @@ -71,7 +55,5 @@ artifacts (such as code repositories). :class:`GitRepoProvider` --------------------------- -A generic repository provider for URLs that point directly to a git repository. - .. autoconfigurable:: GitRepoProvider :members: