From 9e13c3b0cd1e5e19a8df4b10fc0445ecd8bb8641 Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Mon, 13 Jul 2015 10:27:08 -0400 Subject: [PATCH 1/4] DOC: Initial work on Contents API docs. --- docs/source/conf.py | 3 +- docs/source/extending.rst | 143 ++++++++++++++++++++++++++ docs/source/index.rst | 1 + docs/source/public_server.rst | 22 ---- notebook/services/contents/manager.py | 18 ++-- 5 files changed, 156 insertions(+), 31 deletions(-) create mode 100644 docs/source/extending.rst diff --git a/docs/source/conf.py b/docs/source/conf.py index 2007b1d82..c44b636ec 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -41,6 +41,7 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', + 'sphinx.ext.autosummary', ] # Add any paths that contain templates here, relative to this directory. @@ -120,7 +121,7 @@ todo_include_todos = False # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -# html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/source/extending.rst b/docs/source/extending.rst new file mode 100644 index 000000000..128bdb2b9 --- /dev/null +++ b/docs/source/extending.rst @@ -0,0 +1,143 @@ +.. _extending: + +====================== +Extending the Notebook +====================== + +Certain subsystems of the notebook server are designed to be extended or +overridden by users. This document explains the abstractions presented by +these systems and shows how to override the notebook's defaults with your own +custom behavior. + +Contents API +------------ + +.. currentmodule:: notebook.services.contents + +The Jupyter Notebook web application provides a graphical user interface for +creating, opening, renaming, and deleting files in a virtual filesystem. + +The :class:`ContentsManager` class defines an abstract +API for translating these interactions into operations on a particular storage +medium. The default implementation, +:class:`FileContentsManager`, uses the local +filesystem of the server for storage: creating a notebook in the browser +creates a file on disk, deleting a notebook in the browser deletes a file on +disk, and renaming a file in the browser moves a file on disk. + +Data Model +^^^^^^^^^^ + +.. currentmodule:: notebook.services.contents.manager + +API Paths +''''''''' +.. _apipaths: + +ContentsManager methods represent the locations of filesystem resources as +**API-style paths**. Such paths are interpreted as relative to the root +directory of the notebook server. For compatibility across systems, the +following guarantees are made: + +* Paths are always ``unicode``, not ``bytes``. +* Paths are not URL-escaped. +* Paths are always forward-slash (/) delimited, even on Windows. +* Leading and trailing slashes are stripped. For example, ``/foo/bar/buzz/`` + becomes ``foo/bar/buzz``. +* The root directory is represented with the empty string. + +Filesystem Entities +''''''''''''''''''' +.. _notebook models: + +ContentsManager methods represent virtual filesystem entities as dictionaries, +which we refer to as **models**. + +Models may contain the following entries: + ++--------------------+-----------+------------------------------+ +| Key | Type |Info | ++====================+===========+==============================+ +|**name** |unicode |Basename of the entity. | ++--------------------+-----------+------------------------------+ +|**path** |unicode |Full | +| | |(:ref:`API-style`) | +| | |path to the entity. | ++--------------------+-----------+------------------------------+ +|**type** |unicode |The entity type. One of | +| | |``"notebook"``, ``"file"`` or | +| | |``"directory"``. | ++--------------------+-----------+------------------------------+ +|**created** |unicode |Creation date of the entity, | +| | |as ISO-8601 string. | +| | | | ++--------------------+-----------+------------------------------+ +|**last_modified** |unicode |Last modified date of the | +| | |entity, as ISO-8601 string. | +| | | | ++--------------------+-----------+------------------------------+ +|**content** |variable |The "content" of the entity. | +| | |(:ref:`See | +| | |Below`) | ++--------------------+-----------+------------------------------+ +|**mimetype** |unicode or |The mimetype of ``"content"``,| +| |``None`` |if any. (:ref:`See | +| | |Below`) | ++--------------------+-----------+------------------------------+ +|**format** |unicode or |The format of ``"content"``, | +| |``None`` |if any. (:ref:`See | +| | |Below`) | ++--------------------+-----------+------------------------------+ + +.. _modelcontent: + +The **"content"** field + +In certain circumstances, we may not need the full content of an entity to +complete a Contents API request. require the full content of a filesystem +entity. For example, if we want to list the entries in directory ""foo/bar", +we don't want to include the data stored in each file + +A sample model looks as follows: + +.. sourcecode:: python + + { + "name": "My Notebook.ipynb", + "path": "foo/bar/My Notebook.ipynb", + "type": "notebook", + "writable": "true", + "created": "2013-10-01T14:22:36.123456+00:00", + "last_modified": "2013-10-02T11:29:27.616675+00:00", + "mimetype": None, + "content": None, + "format": None, + } + +Writing a Custom ContentsManager +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Required Methods +'''''''''''''''' + +A minimal complete implementation of a custom +:class:`ContentsManager` must implement the following +methods: + +.. autosummary:: + ContentsManager.get + ContentsManager.save + ContentsManager.delete_file + ContentsManager.rename_file + ContentsManager.file_exists + ContentsManager.dir_exists + ContentsManager.is_hidden + +Testing +''''''' +.. currentmodule:: notebook.services.contents.tests + +:mod:`notebook.services.contents.tests` includes several test suites written +against the abstract Contents API. This means that an excellent way to test a +new ContentsManager subclass is to subclass our tests (**!**) to make them use +your ContentsManager. diff --git a/docs/source/index.rst b/docs/source/index.rst index 5ee4296f6..55f8e1166 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -9,5 +9,6 @@ The Jupyter notebook config public_server security + extending development examples/Notebook/Examples and Tutorials Index diff --git a/docs/source/public_server.rst b/docs/source/public_server.rst index f0f8b17d3..80c7d8b24 100644 --- a/docs/source/public_server.rst +++ b/docs/source/public_server.rst @@ -132,28 +132,6 @@ modifying ``jupyter_notebook_config.py``):: c.NotebookApp.base_url = '/ipython/' c.NotebookApp.webapp_settings = {'static_url_prefix':'/ipython/static/'} -Using a different notebook store --------------------------------- - -By default, the notebook server stores the notebook documents that it saves as -files in the working directory of the notebook server, also known as the -``notebook_dir``. This logic is implemented in the -:class:`FileNotebookManager` class. However, the server can be configured to -use a different notebook manager class, which can -store the notebooks in a different format. - -The bookstore_ package currently allows users to store notebooks on Rackspace -CloudFiles or OpenStack Swift based object stores. - -Writing a notebook manager is as simple as extending the base class -:class:`NotebookManager`. The simple_notebook_manager_ provides a great example -of an in memory notebook manager, created solely for the purpose of -illustrating the notebook manager API. - -.. _bookstore: https://github.com/rgbkrk/bookstore - -.. _simple_notebook_manager: https://github.com/khinsen/simple_notebook_manager - Known issues ------------ diff --git a/notebook/services/contents/manager.py b/notebook/services/contents/manager.py index 164079970..76db41c0b 100644 --- a/notebook/services/contents/manager.py +++ b/notebook/services/contents/manager.py @@ -124,7 +124,7 @@ class ContentsManager(LoggingConfigurable): # implemented in subclasses. def dir_exists(self, path): - """Does the API-style path (directory) actually exist? + """Does a directory exist at the given path? Like os.path.isdir @@ -143,7 +143,7 @@ class ContentsManager(LoggingConfigurable): raise NotImplementedError def is_hidden(self, path): - """Does the API style path correspond to a hidden directory or file? + """Is path a hidden directory or file? Parameters ---------- @@ -196,23 +196,25 @@ class ContentsManager(LoggingConfigurable): return self.file_exists(path) or self.dir_exists(path) def get(self, path, content=True, type=None, format=None): - """Get the model of a file or directory with or without content.""" + """Get a file or directory model.""" raise NotImplementedError('must be implemented in a subclass') def save(self, model, path): - """Save the file or directory and return the model with no content. + """ + Save a file or directory model to path. - Save implementations should call self.run_pre_save_hook(model=model, path=path) - prior to writing any data. + Should return the saved model with no content. Save implementations + should call self.run_pre_save_hook(model=model, path=path) prior to + writing any data. """ raise NotImplementedError('must be implemented in a subclass') def delete_file(self, path): - """Delete file or directory by path.""" + """Delete the file or directory at path.""" raise NotImplementedError('must be implemented in a subclass') def rename_file(self, old_path, new_path): - """Rename a file.""" + """Rename a file or directory.""" raise NotImplementedError('must be implemented in a subclass') # ContentsManager API part 2: methods that have useable default From 7e720ab22b614c173ab013b7658cdc8e5896858a Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Fri, 17 Jul 2015 17:29:14 -0400 Subject: [PATCH 2/4] WIP: Moar docs. --- docs/source/extending.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/source/extending.rst b/docs/source/extending.rst index 128bdb2b9..d2ebf25f8 100644 --- a/docs/source/extending.rst +++ b/docs/source/extending.rst @@ -5,25 +5,26 @@ Extending the Notebook ====================== Certain subsystems of the notebook server are designed to be extended or -overridden by users. This document explains the abstractions presented by -these systems and shows how to override the notebook's defaults with your own -custom behavior. +overridden by users. This document explains these systems and shows how to +override the notebook's defaults with your own custom behavior. Contents API ------------ .. currentmodule:: notebook.services.contents -The Jupyter Notebook web application provides a graphical user interface for +The Jupyter Notebook web application provides a graphical interface for creating, opening, renaming, and deleting files in a virtual filesystem. The :class:`ContentsManager` class defines an abstract API for translating these interactions into operations on a particular storage medium. The default implementation, :class:`FileContentsManager`, uses the local -filesystem of the server for storage: creating a notebook in the browser -creates a file on disk, deleting a notebook in the browser deletes a file on -disk, and renaming a file in the browser moves a file on disk. +filesystem of the server for storage and straightforwardly serializes notebooks +into JSON. Users can override these behaviors by supplying custom subclasses +of `ContentsManager` + +This section describes the **Contents API**, Data Model ^^^^^^^^^^ From 0d70a84ef108c05ed751c6556c3ba97d7c9e8018 Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Sat, 25 Jul 2015 19:01:03 -0400 Subject: [PATCH 3/4] DOC: More notes on Contents API. --- docs/source/extending.rst | 172 ++++++++++++++++++++++++++++---------- 1 file changed, 127 insertions(+), 45 deletions(-) diff --git a/docs/source/extending.rst b/docs/source/extending.rst index d2ebf25f8..d15e6c147 100644 --- a/docs/source/extending.rst +++ b/docs/source/extending.rst @@ -22,31 +22,16 @@ medium. The default implementation, :class:`FileContentsManager`, uses the local filesystem of the server for storage and straightforwardly serializes notebooks into JSON. Users can override these behaviors by supplying custom subclasses -of `ContentsManager` +of ContentsManager. -This section describes the **Contents API**, +This section describes the interface implemented by ContentsManager subclasses. +We refer to this interface as the **Contents API**. Data Model ^^^^^^^^^^ .. currentmodule:: notebook.services.contents.manager -API Paths -''''''''' -.. _apipaths: - -ContentsManager methods represent the locations of filesystem resources as -**API-style paths**. Such paths are interpreted as relative to the root -directory of the notebook server. For compatibility across systems, the -following guarantees are made: - -* Paths are always ``unicode``, not ``bytes``. -* Paths are not URL-escaped. -* Paths are always forward-slash (/) delimited, even on Windows. -* Leading and trailing slashes are stripped. For example, ``/foo/bar/buzz/`` - becomes ``foo/bar/buzz``. -* The root directory is represented with the empty string. - Filesystem Entities ''''''''''''''''''' .. _notebook models: @@ -69,55 +54,134 @@ Models may contain the following entries: | | |``"notebook"``, ``"file"`` or | | | |``"directory"``. | +--------------------+-----------+------------------------------+ -|**created** |unicode |Creation date of the entity, | -| | |as ISO-8601 string. | -| | | | +|**created** |datetime |Creation date of the entity. | +--------------------+-----------+------------------------------+ -|**last_modified** |unicode |Last modified date of the | -| | |entity, as ISO-8601 string. | -| | | | +|**last_modified** |datetime |Last modified date of the | +| | |entity. | +--------------------+-----------+------------------------------+ |**content** |variable |The "content" of the entity. | | | |(:ref:`See | | | |Below`) | +--------------------+-----------+------------------------------+ -|**mimetype** |unicode or |The mimetype of ``"content"``,| +|**mimetype** |unicode or |The mimetype of ``content``, | | |``None`` |if any. (:ref:`See | | | |Below`) | +--------------------+-----------+------------------------------+ -|**format** |unicode or |The format of ``"content"``, | +|**format** |unicode or |The format of ``content``, | | |``None`` |if any. (:ref:`See | | | |Below`) | +--------------------+-----------+------------------------------+ .. _modelcontent: +Certain model fields vary in structure depending on the ``type`` field of the +model. There are three model types: **notebook**, **file**, and **directory** . -The **"content"** field +- ``notebook`` models + - The ``format`` field is always ``"json"``. + - The ``mimetype`` field is always ``None``. + - The ``content`` field contains a + :class:`nbformat.notebooknode.NotebookNode` representing the .ipynb file + represented by the model. See the `NBFormat`_ documentation for a full + description. -In certain circumstances, we may not need the full content of an entity to -complete a Contents API request. require the full content of a filesystem -entity. For example, if we want to list the entries in directory ""foo/bar", -we don't want to include the data stored in each file +- ``file`` models + - The ``format`` field is either ``"text"`` or ``"base64"``. + - The ``mimetype`` field is ``text/plain`` for text-format models and + ``application/octet-stream`` for base64-format models. + - The ``content`` field is always of type ``unicode``. For text-format + file models, ``content`` simply contains the file's bytes after decoding + as UTF-8. Non-text (``base64``) files are read as bytes, base64 encoded, + and then decoded as UTF-8. -A sample model looks as follows: +- ``directory`` models + - The ``format`` field is always ``"json"``. + - The ``mimetype`` field is always ``None``. + - The ``content`` field contains a list of :ref:`content-free` + models representing the entities in the directory. + +.. note:: + + .. _contentfree: + + In certain circumstances, we don't need the full content of an entity to + complete a Contents API request. In such cases, we omit the ``mimetype``, + ``content``, and ``format`` keys from the model. This most commonly occurs + when listing a directory, in which circumstance we represent files within + the directory as content-less models to avoid having to recursively traverse + and serialize the entire filesystem. + +**Sample Models** .. sourcecode:: python - { - "name": "My Notebook.ipynb", - "path": "foo/bar/My Notebook.ipynb", - "type": "notebook", - "writable": "true", - "created": "2013-10-01T14:22:36.123456+00:00", - "last_modified": "2013-10-02T11:29:27.616675+00:00", - "mimetype": None, - "content": None, - "format": None, - } + # Notebook Model with Content + { + 'content': { + 'metadata': {}, + 'nbformat': 4, + 'nbformat_minor': 0, + 'cells': [ + { + 'cell_type': 'markdown', + 'metadata': {}, + 'source': 'Some **Markdown**', + }, + ], + }, + 'created': datetime(2015, 7, 25, 19, 50, 19, 19865), + 'format': 'json', + 'last_modified': datetime(2015, 7, 25, 19, 50, 19, 19865), + 'mimetype': None, + 'name': 'a.ipynb', + 'path': 'foo/a.ipynb', + 'type': 'notebook', + 'writable': True, + } + + # Notebook Model without Content + { + 'content': None, + 'created': datetime.datetime(2015, 7, 25, 20, 17, 33, 271931), + 'format': None, + 'last_modified': datetime.datetime(2015, 7, 25, 20, 17, 33, 271931), + 'mimetype': None, + 'name': 'a.ipynb', + 'path': 'foo/a.ipynb', + 'type': 'notebook', + 'writable': True + } + + +API Paths +''''''''' +.. _apipaths: + +ContentsManager methods represent the locations of filesystem resources as +**API-style paths**. Such paths are interpreted as relative to the root +directory of the notebook server. For compatibility across systems, the +following guarantees are made: + +* Paths are always ``unicode``, not ``bytes``. +* Paths are not URL-escaped. +* Paths are always forward-slash (/) delimited, even on Windows. +* Leading and trailing slashes are stripped. For example, ``/foo/bar/buzz/`` + becomes ``foo/bar/buzz``. +* The empty string (``""``) represents the root directory. + Writing a Custom ContentsManager ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The default ContentsManager is designed for users running the notebook as an +application on a personal computer. It stores notebooks as .ipynb files on the +local filesystem, and it maps files and directories in the Notebook UI to files +and directories on disk. It is possible to override how notebooks are stored +by implementing your own custom subclass of ``ContentsManager``. For example, +if you deploy the notebook in a context where you don't trust or don't have +access to the filesystem of the notebook server, it's possible to write your +own ContentsManager that stores notebooks and files in a database. + + Required Methods '''''''''''''''' @@ -134,11 +198,29 @@ methods: ContentsManager.dir_exists ContentsManager.is_hidden + +Customizing Checkpoints +''''''''''''''''''''''' + +TODO: + + Testing ''''''' .. currentmodule:: notebook.services.contents.tests :mod:`notebook.services.contents.tests` includes several test suites written against the abstract Contents API. This means that an excellent way to test a -new ContentsManager subclass is to subclass our tests (**!**) to make them use -your ContentsManager. +new ContentsManager subclass is to subclass our tests to make them use your +ContentsManager. + +.. note:: + +PGContents_ is an example of a complete implementation of a custom +``ContentsManager``. It stores notebooks and files in PostgreSQL_ and encodes +directories as SQL relations. PGContents also provides an example of how to +re-use the notebook's tests. + +.. _NBFormat: http://nbformat.readthedocs.org/en/latest/index.html +.. _PGContents: https://github.com/quantopian/pgcontents +.. _PostgreSQL: http://www.postgresql.org/ From e76548a96a4bd511c4b8f086a064377ae484ca7c Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Sat, 25 Jul 2015 19:05:30 -0400 Subject: [PATCH 4/4] MAINT: Simpler way of writing classes. --- docs/source/extending.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/extending.rst b/docs/source/extending.rst index d15e6c147..ba921c13d 100644 --- a/docs/source/extending.rst +++ b/docs/source/extending.rst @@ -16,10 +16,10 @@ Contents API The Jupyter Notebook web application provides a graphical interface for creating, opening, renaming, and deleting files in a virtual filesystem. -The :class:`ContentsManager` class defines an abstract +The :class:`~manager.ContentsManager` class defines an abstract API for translating these interactions into operations on a particular storage medium. The default implementation, -:class:`FileContentsManager`, uses the local +:class:`~filemanager.FileContentsManager`, uses the local filesystem of the server for storage and straightforwardly serializes notebooks into JSON. Users can override these behaviors by supplying custom subclasses of ContentsManager. @@ -186,7 +186,7 @@ Required Methods '''''''''''''''' A minimal complete implementation of a custom -:class:`ContentsManager` must implement the following +:class:`~manager.ContentsManager` must implement the following methods: .. autosummary::