First pass migrating from jupyter_cms

* Change handler for consistency with nbconvert
* Change bundler function API to take ContentManager models
* Change CLI to work as jupyter bundler
* Change UI to fit into existing template / JS structure

(c) Copyright IBM Corp. 2016

notebook/bundler/handlers.py

@@ -0,0 +1,242 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import os
+import shutil
+import errno
+import nbformat
+import fnmatch
+import glob
+from notebook.utils import url_path_join, url2path
+from notebook.base.handlers import IPythonHandler, path_regex
+from notebook.services.config import ConfigManager
+from ipython_genutils.importstring import import_item
+from tornado import web, gen
+
+class BundlerTools(object):
+    '''Set of common tools to aid bundler implementations.'''
+    def get_file_references(self, abs_nb_path, version):
+        '''
+        Gets a list of files referenced either in Markdown fenced code blocks
+        or in HTML comments from the notebook. Expands patterns expressed in 
+        gitignore syntax (https://git-scm.com/docs/gitignore). Returns the 
+        fully expanded list of filenames relative to the notebook dirname.
+
+        NOTE: Temporarily changes the current working directory when called.
+
+        :param abs_nb_path: Absolute path of the notebook on disk
+        :param version: Version of the notebook document format to use
+        :returns: List of filename strings relative to the notebook path
+        '''
+        ref_patterns = self.get_reference_patterns(abs_nb_path, version)
+        expanded = self.expand_references(os.path.dirname(abs_nb_path), ref_patterns)
+        return expanded
+
+    def get_reference_patterns(self, abs_nb_path, version):
+        '''
+        Gets a list of reference patterns either in Markdown fenced code blocks
+        or in HTML comments from the notebook.
+
+        :param abs_nb_path: Absolute path of the notebook on disk
+        :param version: Version of the notebook document format to use
+        :returns: List of pattern strings from the notebook
+        '''
+        notebook = nbformat.read(abs_nb_path, version)
+        referenced_list = []
+        for cell in notebook.cells:
+            references = self.get_cell_reference_patterns(cell)
+            if references:
+                referenced_list = referenced_list + references
+        return referenced_list
+
+    def get_cell_reference_patterns(self, cell):
+        '''
+        Retrieves the list of references from a single notebook cell. Looks for
+        fenced code blocks or HTML comments in Markdown cells, e.g.,
+
+        ```
+        some.csv
+        foo/
+        !foo/bar
+        ```
+
+        or 
+
+        <!--associate:
+        some.csv
+        foo/
+        !foo/bar
+        -->
+
+        :param cell: Notebook cell object
+        :returns: List of strings
+        '''
+        referenced = []
+        # invisible after execution: unrendered HTML comment
+        if cell.get('cell_type').startswith('markdown') and cell.get('source').startswith('<!--associate:'):
+            lines = cell.get('source')[len('<!--associate:'):].splitlines()
+            for line in lines:
+                if line.startswith('-->'):
+                    break
+                # Trying to go out of the current directory leads to
+                # trouble when deploying
+                if line.find('../') < 0 and not line.startswith('#'):
+                    referenced.append(line)
+        # visible after execution: rendered as a code element within a pre element
+        elif cell.get('cell_type').startswith('markdown') and cell.get('source').find('```') >= 0:
+            source = cell.get('source')
+            offset = source.find('```')
+            lines = source[offset + len('```'):].splitlines()
+            for line in lines:
+                if line.startswith('```'):
+                    break
+                # Trying to go out of the current directory leads to
+                # trouble when deploying
+                if line.find('../') < 0 and not line.startswith('#'):
+                    referenced.append(line)
+
+        # Clean out blank references
+        return [ref for ref in referenced if ref.strip()]
+
+    def expand_references(self, root_path, references):
+        '''
+        Expands a set of reference patterns by evaluating them against the
+        given root directory. Expansions are performed against patterns 
+        expressed in the same manner as in gitignore 
+        (https://git-scm.com/docs/gitignore).
+
+        :param root_path: Assumed root directory for the patterns
+        :param references: List of reference patterns
+        :returns: List of filename strings relative to the root path
+        '''
+        globbed = []
+        negations = []
+        must_walk = []
+        for pattern in references:
+            if pattern and pattern.find('/') < 0:
+                # simple shell glob
+                cwd = os.getcwd()
+                os.chdir(root_path)
+                if pattern.startswith('!'):
+                    negations = negations + glob.glob(pattern[1:])
+                else:
+                    globbed = globbed + glob.glob(pattern)
+                os.chdir(cwd)
+            elif pattern:
+                must_walk.append(pattern)
+
+        for pattern in must_walk:
+            pattern_is_negation = pattern.startswith('!')
+            if pattern_is_negation:
+                testpattern = pattern[1:]
+            else:
+                testpattern = pattern
+            for root, _, filenames in os.walk(root_path):
+                for filename in filenames:
+                    joined = os.path.join(root[len(root_path) + 1:], filename)
+                    if testpattern.endswith('/'):
+                        if joined.startswith(testpattern):
+                            if pattern_is_negation:
+                                negations.append(joined)
+                            else:
+                                globbed.append(joined)
+                    elif testpattern.find('**') >= 0:
+                        # path wildcard
+                        ends = testpattern.split('**')
+                        if len(ends) == 2:
+                            if joined.startswith(ends[0]) and joined.endswith(ends[1]):
+                                if pattern_is_negation:
+                                    negations.append(joined)
+                                else:
+                                    globbed.append(joined)
+                    else:
+                        # segments should be respected
+                        if fnmatch.fnmatch(joined, testpattern):
+                            if pattern_is_negation:
+                                negations.append(joined)
+                            else:
+                                globbed.append(joined)
+
+        for negated in negations:
+            try:
+                globbed.remove(negated)
+            except ValueError as err:
+                pass
+        return set(globbed)
+
+    def copy_filelist(self, src, dst, src_relative_filenames):
+        '''
+        Copies the given list of files, relative to src, into dst, creating
+        directories along the way as needed and ignore existence errors.
+        Skips any files that do not exist. Does not create empty directories
+        from src in dst.
+
+        :param src: Root of the source directory
+        :param dst: Root of the destination directory
+        :param src_relative_filenames: List of filename relative to src
+        '''
+        for filename in src_relative_filenames:
+            # Only consider the file if it exists in src
+            if os.path.isfile(os.path.join(src, filename)):
+                parent_relative = os.path.dirname(filename)
+                if parent_relative:
+                    # Make sure the parent directory exists
+                    parent_dst = os.path.join(dst, parent_relative)
+                    try:
+                        os.makedirs(parent_dst)
+                    except OSError as exc:
+                        if exc.errno == errno.EEXIST:
+                            pass
+                        else:
+                            raise exc
+                shutil.copy2(os.path.join(src, filename), os.path.join(dst, filename))
+
+class BundlerHandler(IPythonHandler):
+    def initialize(self):
+        # Create common tools for bundler plugin to use
+        self.tools = BundlerTools()
+
+    def get_bundler(self, bundler_id):
+        '''
+        :param bundler_id: Unique ID within the notebook/jupyter_cms_bundlers
+        config section.
+        :returns: Dict of bundler metadata with keys label, group, module_name
+        :raises KeyError: If the bundler is not registered
+        '''
+        cm = ConfigManager()
+        return cm.get('notebook').get('bundlers', {})[bundler_id]
+
+    @web.authenticated
+    @gen.coroutine
+    def get(self, path):
+        """Bundle the given notebook.
+        """
+        bundler_id = self.get_query_argument('bundler')
+        model = self.contents_manager.get(path=url2path(path))
+
+        try:
+            bundler = self.get_bundler(bundler_id)
+        except KeyError:
+            raise web.HTTPError(404, 'Bundler %s not found' % bundler_id)
+        
+        module_name = bundler['module_name']
+        try:
+            # no-op in python3, decode error in python2
+            module_name = str(module_name)
+        except UnicodeEncodeError:
+            # Encode unicode as utf-8 in python2 else import_item fails
+            module_name = module_name.encode('utf-8')
+        
+        try:
+            bundler_mod = import_item(module_name)
+        except ImportError:
+            raise web.HTTPError(500, 'Could not import bundler %s ' % bundler_id)
+
+        # Let the bundler respond in any way it sees fit and assume it will
+        # finish the request
+        yield gen.maybe_future(bundler_mod.bundle(self, model))
+
+_bundler_id_regex = r'(?P<bundler_id>[A-Za-z0-9_]+)'
+
+default_handlers = [
+    (r"/bundle/(.*)", BundlerHandler)
+]

notebook/bundlerextensions.py

@@ -0,0 +1,259 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import sys
+import os
+
+from .nbextensions import (BaseNBExtensionApp, _get_config_dir)    
+from ._version import __version__
+
+from traitlets.config.manager import BaseJSONConfigManager
+from traitlets.utils.importstring import import_item
+from traitlets import Bool
+
+BUNDLER_SECTION = "notebook"
+BUNDLER_SUBSECTION = "bundlers"
+
+def _get_bundler_metadata(module):
+    """Gets the list of bundlers associated with a Python package.
+    
+    Returns a tuple of (the module, [{
+        'name': 'unique name of the bundler',
+        'label': 'file menu item label for the bundler',
+        'module_name': 'dotted package/module name containing the bundler',
+        'group': 'download or deploy parent menu item'
+    }])
+    
+    Parameters
+    ----------
+
+    module : str
+        Importable Python module exposing the
+        magic-named `_jupyter_bundler_paths` function    
+    """
+    m = import_item(module)
+    if not hasattr(m, '_jupyter_bundler_paths'):
+        raise KeyError('The Python module {} does not contain a valid bundler'.format(module))
+    bundlers = m._jupyter_bundler_paths()
+    return m, bundlers
+
+def _set_bundler_state(name, label, module_name, group, state,
+                       user=True, sys_prefix=False, logger=None):
+    """Set whether a bundler is enabled or disabled.
+    
+    Returns True if the final state is the one requested.
+    
+    Parameters
+    ----------
+    name : string
+        Unique name of the bundler
+    label : string
+        Human-readable label for the bundler menu item in the notebook UI
+    module_name : string
+        Dotted module/package name containing the bundler
+    group : string
+        'download' or 'deploy' indicating the parent menu containing the label
+    state : bool
+        The state in which to leave the extension
+    user : bool [default: True]
+        Whether to update the user's .jupyter/nbconfig directory
+    sys_prefix : bool [default: False]
+        Whether to update the sys.prefix, i.e. environment. Will override
+        `user`.
+    logger : Jupyter logger [optional]
+        Logger instance to use
+    """
+    user = False if sys_prefix else user
+    config_dir = os.path.join(
+        _get_config_dir(user=user, sys_prefix=sys_prefix), 'nbconfig')
+    cm = BaseJSONConfigManager(config_dir=config_dir)
+    
+    if logger:
+        logger.info("{} {} bundler {}...".format(
+            "Enabling" if state else "Disabling",
+            name,
+            module_name
+        ))
+    
+    if state:
+        cm.update(BUNDLER_SECTION, {
+            BUNDLER_SUBSECTION: {
+                name: {
+                    "label": label,
+                    "module_name": module_name,
+                    "group" : group
+                }
+            }
+        })
+    else:
+        cm.update(BUNDLER_SECTION, {
+            BUNDLER_SUBSECTION: {
+                name: None
+            }
+        })
+
+    return (cm.get(BUNDLER_SECTION)
+              .get(BUNDLER_SUBSECTION, {})
+              .get(name) is not None) == state
+    
+
+def _set_bundler_state_python(state, module, user, sys_prefix, logger=None):
+    """Enables or disables bundlers defined in a Python package.
+    
+    Returns a list of whether the state was achieved for each bundler.
+    
+    Parameters
+    ----------
+    state : Bool
+        Whether the extensions should be enabled
+    module : str
+        Importable Python module exposing the
+        magic-named `_jupyter_bundler_paths` function
+    user : bool
+        Whether to enable in the user's nbconfig directory.
+    sys_prefix : bool
+        Enable/disable in the sys.prefix, i.e. environment
+    logger : Jupyter logger [optional]
+        Logger instance to use
+    """
+    m, bundlers = _get_bundler_metadata(module)
+    return [_set_bundler_state(name=bundler["name"],
+                               label=bundler["label"],
+                               module_name=bundler["module_name"],
+                               group=bundler["group"],
+                               state=state,
+                               user=user, sys_prefix=sys_prefix,
+                               logger=logger)
+            for bundler in bundlers]
+
+def enable_bundler_python(module, user=True, sys_prefix=False, logger=None):
+    """Enables bundlers defined in a Python package.
+    
+    Returns whether each bundle defined in the packaged was enabled or not.
+    
+    Parameters
+    ----------
+    module : str
+        Importable Python module exposing the
+        magic-named `_jupyter_bundler_paths` function
+    user : bool [default: True]
+        Whether to enable in the user's nbconfig directory.
+    sys_prefix : bool [default: False]
+        Whether to enable in the sys.prefix, i.e. environment. Will override
+        `user`
+    logger : Jupyter logger [optional]
+        Logger instance to use
+    """
+    return _set_bundler_state_python(True, module, user, sys_prefix,
+                                     logger=logger)
+    
+def disable_bundler_python(module, user=True, sys_prefix=False, logger=None):
+    """Disables bundlers defined in a Python package.
+    
+    Returns whether each bundle defined in the packaged was enabled or not.
+    
+    Parameters
+    ----------
+    module : str
+        Importable Python module exposing the
+        magic-named `_jupyter_bundler_paths` function
+    user : bool [default: True]
+        Whether to enable in the user's nbconfig directory.
+    sys_prefix : bool [default: False]
+        Whether to enable in the sys.prefix, i.e. environment. Will override
+        `user`
+    logger : Jupyter logger [optional]
+        Logger instance to use
+    """
+    return _set_bundler_state_python(False, module, user, sys_prefix,
+                                     logger=logger)
+
+class ToggleNBBundlerApp(BaseNBExtensionApp):
+    """A base class for apps that enable/disable bundlers"""
+    name = "jupyter bundler enable/disable"
+    version = __version__
+    description = "Enable/disable a bundler in configuration."
+
+    user = Bool(True, config=True, help="Apply the configuration only for the current user (default)")
+    
+    _toggle_value = None
+    
+    def _config_file_name_default(self):
+        """The default config file name."""
+        return 'jupyter_notebook_config'
+    
+    def toggle_bundler_python(self, module):
+        """Toggle some extensions in an importable Python module.
+
+        Returns a list of booleans indicating whether the state was changed as
+        requested.
+
+        Parameters
+        ----------
+        module : str
+            Importable Python module exposing the
+            magic-named `_jupyter_bundler_paths` function
+        """
+        toggle = (enable_bundler_python if self._toggle_value
+                  else disable_bundler_python)
+        return toggle(module,
+                      user=self.user,
+                      sys_prefix=self.sys_prefix,
+                      logger=self.log)
+
+    def start(self):
+        if not self.extra_args:
+            sys.exit('Please specify an nbextension/package to enable or disable')
+        elif len(self.extra_args) > 1:
+            sys.exit('Please specify one nbextension/package at a time')
+        if self.python:
+            self.toggle_bundler_python(self.extra_args[0])
+        else:
+            raise NotImplementedError('Cannot install bundlers from non-Python packages')            
+
+class EnableNBBundlerApp(ToggleNBBundlerApp):
+    """An App that enables bundlers"""
+    name = "jupyter bundler enable"
+    description = """
+    Enable a bundler in frontend configuration.
+    
+    Usage
+        jupyter bundler enable [--system|--sys-prefix]
+    """
+    _toggle_value = True
+    
+class DisableNBBundlerApp(ToggleNBBundlerApp):
+    """An App that disables bundlers"""
+    name = "jupyter bundler disable"
+    description = """
+    Disable a bundler in frontend configuration.
+    
+    Usage
+        jupyter bundler disable [--system|--sys-prefix]
+    """
+    _toggle_value = None
+
+class NBBundlerApp(BaseNBExtensionApp):
+    """Base jupyter bundler command entry point"""
+    name = "jupyter bundler"
+    version = __version__
+    description = "Work with Jupyter notebook bundlers"
+    examples = """
+jupyter bundler enable --py <packagename>     # enable all bundlers in a Python package
+jupyter bundler disable --py <packagename>    # disable all bundlers in a Python package
+"""
+
+    subcommands = dict(
+        enable=(EnableNBBundlerApp, "Enable a bundler"),
+        disable=(DisableNBBundlerApp, "Disable a bundler")
+    )
+
+    def start(self):
+        """Perform the App's functions as configured"""
+        super(NBBundlerApp, self).start()
+
+        # The above should have called a subcommand and raised NoStart; if we
+        # get here, it didn't, so we should self.log.info a message.
+        subcmds = ", ".join(sorted(self.subcommands))
+        sys.exit("Please supply at least one subcommand: %s" % subcmds)
+
+main = NBBundlerApp.launch_instance

notebook/notebookapp.py

@@ -260,6 +260,7 @@ class NotebookWebApplication(web.Application):
         handlers.extend(load_handlers('files.handlers'))
         handlers.extend(load_handlers('notebook.handlers'))
         handlers.extend(load_handlers('nbconvert.handlers'))
+        handlers.extend(load_handlers('bundler.handlers'))
         handlers.extend(load_handlers('kernelspecs.handlers'))
         handlers.extend(load_handlers('edit.handlers'))
         handlers.extend(load_handlers('services.api.handlers'))

notebook/static/notebook/js/main.js

@@ -141,7 +141,8 @@ require([
         events: events,
         save_widget: save_widget,
         quick_help: quick_help,
-        actions: acts},
+        actions: acts,
+        config: config_section},
         common_options));
     var notification_area = new notificationarea.NotebookNotificationArea(
         '#notification_area', {

notebook/static/notebook/js/menubar.js

@@ -29,6 +29,7 @@ define([
          *          base_url : string
          *          notebook_path : string
          *          notebook_name : string
+         *          config: ConfigSection instance
          */
         options = options || {};
         this.base_url = options.base_url || utils.get_body_data("baseUrl");
@@ -40,6 +41,7 @@ define([
         this.save_widget = options.save_widget;
         this.quick_help = options.quick_help;
         this.actions = options.actions;
+        this.config = options.config;
 
         try {
             this.tour = new tour.Tour(this.notebook, this.events);
@@ -51,6 +53,7 @@ define([
         if (this.selector !== undefined) {
             this.element = $(selector);
             this.style();
+            this.add_bundler_items();
             this.bind_events();
         }
     };
@@ -66,6 +69,63 @@ define([
             }
         );
     };
+    
+    MenuBar.prototype.add_bundler_items = function() {
+        var that = this;
+        this.config.loaded.then(function() {
+            var bundlers = that.config.data.bundlers;
+            if(bundlers) {
+                // TODO: maybe sort by label
+                for(var bundler_id in bundlers) {
+                    var bundler = bundlers[bundler_id];
+                    var group = that.element.find('#'+bundler.group+'_menu')
+                    
+                    // Basic validation to ensure valid menu options
+                    if(!group.length) {
+                        console.warn('unknown group', bundler.group, 'for bundler ID', bundler_id, '; skipping');
+                        continue;
+                    } else if(!bundler.label) {
+                        console.warn('no label for bundler ID', bundler_id, '; skipping');
+                        continue;
+                    }
+                    
+                    // New menu item in the right submenu to trigger that._bundle
+                    group.parent().removeClass('hidden');
+                    var $li = $('<li>')
+                        .appendTo(group);
+                    $('<a>')
+                        .attr('href', '#')
+                        .text(bundler.label)
+                        .appendTo($li)
+                        .on('click', that._bundle.bind(that, bundler_id))
+                        .appendTo($li);
+                }
+            }
+        });
+    };
+
+    MenuBar.prototype._new_window = function(url) {
+        var w = window.open('', IPython._target);
+        if (this.notebook.dirty && this.notebook.writable) {
+            this.notebook.save_notebook().then(function() {
+                w.location = url;
+            });
+        } else {
+            w.location = url;
+        }
+    };
+    
+    MenuBar.prototype._bundle = function(bundler_id) {
+        // Read notebook path and base url here in case they change
+        var notebook_path = utils.encode_uri_components(this.notebook.notebook_path);
+        var url = utils.url_path_join(
+            this.base_url,
+            'bundle',
+            notebook_path
+        ) + '?bundler=' + utils.encode_uri_components(bundler_id);
+
+        this._new_window(url);
+    };
 
     MenuBar.prototype._nbconvert = function (format, download) {
         download = download || false;
@@ -77,14 +137,7 @@ define([
             notebook_path
         ) + "?download=" + download.toString();
         
-        var w = window.open('', IPython._target);
-        if (this.notebook.dirty && this.notebook.writable) {
-            this.notebook.save_notebook().then(function() {
-                w.location = url;
-            });
-        } else {
-            w.location = url;
-        }
+        this._new_window(url);
     };
 
     MenuBar.prototype._size_header = function() {
@@ -154,7 +207,6 @@ define([
             that._nbconvert('script', true);
         });
 
-
         this.events.on('trust_changed.Notebook', function (event, trusted) {
             if (trusted) {
                 that.element.find('#trust_notebook')

notebook/templates/notebook.html

@@ -103,7 +103,7 @@ data-notebook-path="{{notebook_path | urlencode}}"
                         <li class="divider"></li>
                         <li id="print_preview"><a href="#">Print Preview</a></li>
                         <li class="dropdown-submenu"><a href="#">Download as</a>
-                            <ul class="dropdown-menu">
+                            <ul id="download_menu" class="dropdown-menu">
                                 <li id="download_ipynb"><a href="#">Notebook (.ipynb)</a></li>
                                 <li id="download_script"><a href="#">Script</a></li>
                                 <li id="download_html"><a href="#">HTML (.html)</a></li>
@@ -112,6 +112,9 @@ data-notebook-path="{{notebook_path | urlencode}}"
                                 <li id="download_pdf"><a href="#">PDF via LaTeX (.pdf)</a></li>
                             </ul>
                         </li>
+                        <li class="dropdown-submenu hidden"><a href="#">Deploy as</a>
+                            <ul id="deploy_menu" class="dropdown-menu"></ul>
+                        </li>
                         <li class="divider"></li>
                         <li id="trust_notebook"
                             title="Trust the output of this notebook">

scripts/jupyter-bundler

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+from notebook.bundlerextensions import main
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -180,6 +180,7 @@ if 'setuptools' in sys.modules:
             'jupyter-notebook = notebook.notebookapp:main',
             'jupyter-nbextension = notebook.nbextensions:main',
             'jupyter-serverextension = notebook.serverextensions:main',
+            'jupyter-bundler = notebook.bundlerextensions:main',
         ]
     }
     setup_args.pop('scripts', None)