From 22c5e3ec2b4ae37de036f17fdd4d7a0cbf40e2f6 Mon Sep 17 00:00:00 2001 From: MinRK Date: Thu, 30 Jan 2014 15:00:43 -0800 Subject: [PATCH 01/11] mention that pexpect ships in IPython.external and only check for it on posix --- setupbase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setupbase.py b/setupbase.py index b5c9b2089..f544be6a5 100644 --- a/setupbase.py +++ b/setupbase.py @@ -452,7 +452,8 @@ def check_for_dependencies(): check_for_sphinx() check_for_pygments() check_for_nose() - check_for_pexpect() + if os.name == 'posix': + check_for_pexpect() check_for_pyzmq() check_for_tornado() check_for_readline() From 29b1aa4c93a2694a9453e22f3b2ff369deb0f6d6 Mon Sep 17 00:00:00 2001 From: MinRK Date: Thu, 30 Jan 2014 15:03:12 -0800 Subject: [PATCH 02/11] only put Python major version in wheel tags results in 'py2-none-any' instead of 'py27-none-any'. This is supported by pip and PEP 425, but not available from bdist_wheel by default. --- setupbase.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/setupbase.py b/setupbase.py index f544be6a5..1dd6c57a2 100644 --- a/setupbase.py +++ b/setupbase.py @@ -556,6 +556,30 @@ def require_submodules(command): command.run(self) return DecoratedCommand +#--------------------------------------------------------------------------- +# bdist related +#--------------------------------------------------------------------------- + +def get_bdist_wheel(): + """Construct bdist_wheel command for building wheels + + Constructs py2-none-any tag, instead of py2.7-none-any + """ + class RequiresWheel(Command): + def run(self): + print("bdist_wheel requires the wheel package") + if 'setuptools' not in sys.modules: + return RequiresWheel + else: + try: + from wheel.bdist_wheel import bdist_wheel + except ImportError: + return RequiresWheel + class bdist_wheel_tag(bdist_wheel): + def get_tag(self): + return ('py%i' % sys.version_info[0], 'none', 'any') + return bdist_wheel_tag + #--------------------------------------------------------------------------- # Notebook related #--------------------------------------------------------------------------- From d8ade7980da3de18790989caa86c6edfeb0c3747 Mon Sep 17 00:00:00 2001 From: MinRK Date: Thu, 30 Jan 2014 18:44:29 -0800 Subject: [PATCH 03/11] platform-dependent requirements in bdist_wheel --- setupbase.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/setupbase.py b/setupbase.py index 1dd6c57a2..cc92088e5 100644 --- a/setupbase.py +++ b/setupbase.py @@ -572,12 +572,41 @@ def get_bdist_wheel(): return RequiresWheel else: try: - from wheel.bdist_wheel import bdist_wheel + from wheel.bdist_wheel import bdist_wheel, read_pkg_info, write_pkg_info except ImportError: return RequiresWheel + class bdist_wheel_tag(bdist_wheel): + def get_tag(self): return ('py%i' % sys.version_info[0], 'none', 'any') + + def add_requirements(self, metadata_path): + """transform platform-dependent requirements""" + pkg_info = read_pkg_info(metadata_path) + # pkg_info is an email.Message object (?!) + # we have to remove the unconditional 'readline' and/or 'pyreadline' entries + # and transform them to conditionals + requires = pkg_info.get_all('Requires-Dist') + del pkg_info['Requires-Dist'] + def _remove_startswith(lis, prefix): + """like list.remove, but with startswith instead of ==""" + found = False + for idx, item in enumerate(lis): + if item.startswith(prefix): + found = True + break + if found: + lis.pop(idx) + + for pkg in ("readline", "pyreadline"): + _remove_startswith(requires, pkg) + requires.append("readline; sys.platform == 'darwin'") + requires.append("pyreadline (>=2.0); sys.platform == 'win32'") + for r in requires: + pkg_info['Requires-Dist'] = r + write_pkg_info(metadata_path, pkg_info) + return bdist_wheel_tag #--------------------------------------------------------------------------- From 4975f2db6bb98505ce852fef5d9e0d9d38f2cd7e Mon Sep 17 00:00:00 2001 From: MinRK Date: Thu, 30 Jan 2014 18:46:23 -0800 Subject: [PATCH 04/11] always construct requirements --- setup.py | 64 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/setup.py b/setup.py index 5c654008e..e74cfa128 100755 --- a/setup.py +++ b/setup.py @@ -67,6 +67,7 @@ from setupbase import ( update_submodules, require_submodules, UpdateSubmodules, + get_bdist_wheel, CompileCSS, JavascriptVersion, install_symlinked, @@ -242,8 +243,8 @@ setup_args['cmdclass'] = { # For some commands, use setuptools. Note that we do NOT list install here! # If you want a setuptools-enhanced install, just run 'setupegg.py install' needs_setuptools = set(('develop', 'release', 'bdist_egg', 'bdist_rpm', - 'bdist', 'bdist_dumb', 'bdist_wininst', 'install_egg_info', - 'egg_info', 'easy_install', 'upload', + 'bdist', 'bdist_dumb', 'bdist_wininst', 'bdist_wheel', + 'egg_info', 'easy_install', 'upload', 'install_egg_info', )) if sys.platform == 'win32': # Depend on setuptools for install on *Windows only* @@ -259,43 +260,42 @@ if len(needs_setuptools.intersection(sys.argv)) > 0: # specific to setup setuptools_extra_args = {} +# setuptools requirements + +extras_require = dict( + parallel = 'pyzmq>=2.1.11', + qtconsole = ['pyzmq>=2.1.11', 'pygments'], + zmq = 'pyzmq>=2.1.11', + doc = ['Sphinx>=1.1', 'numpydoc'], + test = 'nose>=0.10.1', + notebook = ['tornado>=3.1', 'pyzmq>=2.1.11', 'jinja2'], + nbconvert = ['pygments', 'jinja2', 'Sphinx>=0.3'] +) +everything = set() +for deps in extras_require.values(): + if not isinstance(deps, list): + deps = [deps] + for dep in deps: + everything.add(dep) +extras_require['all'] = everything +install_requires = [] +if sys.platform == 'darwin': + install_requires.append('readline') +elif sys.platform.startswith('win'): + # Pyreadline has unicode and Python 3 fixes in 2.0 + install_requires.append('pyreadline>=2.0') + + if 'setuptools' in sys.modules: # setup.py develop should check for submodules from setuptools.command.develop import develop setup_args['cmdclass']['develop'] = require_submodules(develop) + setup_args['cmdclass']['bdist_wheel'] = get_bdist_wheel() setuptools_extra_args['zip_safe'] = False setuptools_extra_args['entry_points'] = {'console_scripts':find_entry_points()} - setup_args['extras_require'] = dict( - parallel = 'pyzmq>=2.1.11', - qtconsole = ['pyzmq>=2.1.11', 'pygments'], - zmq = 'pyzmq>=2.1.11', - doc = ['Sphinx>=1.1', 'numpydoc'], - test = 'nose>=0.10.1', - notebook = ['tornado>=3.1', 'pyzmq>=2.1.11', 'jinja2'], - nbconvert = ['pygments', 'jinja2', 'Sphinx>=0.3'] - ) - everything = set() - for deps in setup_args['extras_require'].values(): - if not isinstance(deps, list): - deps = [deps] - for dep in deps: - everything.add(dep) - setup_args['extras_require']['all'] = everything - - requires = setup_args.setdefault('install_requires', []) - setupext.display_status = False - if not setupext.check_for_readline(): - if sys.platform == 'darwin': - requires.append('readline') - elif sys.platform.startswith('win'): - # Pyreadline 64 bit windows issue solved in versions >=1.7.1 - # Also solves issues with some older versions of pyreadline that - # satisfy the unconstrained depdendency. - requires.append('pyreadline>=1.7.1') - else: - pass - # do we want to install readline here? + setup_args['extras_require'] = extras_require + requires = setup_args['install_requires'] = install_requires # Script to be run by the windows binary installer after the default setup # routine, to add shortcuts and similar windows-only things. Windows From 143ab47bfa85588bb0a403564532603865dcf83f Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 31 Jan 2014 11:31:24 -0800 Subject: [PATCH 05/11] whitelist installed components we don't need most of what's in components to be installed. Much of it is included just for sdists or development. --- setupbase.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/setupbase.py b/setupbase.py index cc92088e5..0b8185619 100644 --- a/setupbase.py +++ b/setupbase.py @@ -127,23 +127,44 @@ def find_package_data(): # This is not enough for these things to appear in an sdist. # We need to muck with the MANIFEST to get this to work - # exclude static things that we don't ship (e.g. mathjax) - excludes = ['mathjax'] + # exclude components from the walk, + # we will build the components separately + excludes = ['components'] # add 'static/' prefix to exclusions, and tuplify for use in startswith - excludes = tuple([os.path.join('static', ex) for ex in excludes]) + excludes = tuple([pjoin('static', ex) for ex in excludes]) # walk notebook resources: cwd = os.getcwd() os.chdir(os.path.join('IPython', 'html')) - static_walk = list(os.walk('static')) static_data = [] - for parent, dirs, files in static_walk: + for parent, dirs, files in os.walk('static'): if parent.startswith(excludes): continue for f in files: - static_data.append(os.path.join(parent, f)) - + static_data.append(pjoin(parent, f)) + components = pjoin("static", "components") + # select the components we actually need to install + # (there are lots of resources we bundle for sdist-reasons that we don't actually use) + static_data.extend([ + pjoin(components, "backbone", "backbone-min.js"), + pjoin(components, "bootstrap", "bootstrap", "js", "bootstrap.min.js"), + pjoin(components, "font-awesome", "build", "assets", "font", "*.*"), + pjoin(components, "highlight.js", "build", "highlight.pack.js"), + pjoin(components, "jquery", "jquery.min.js"), + pjoin(components, "jquery-ui", "ui", "minified", "jquery-ui.min.js"), + pjoin(components, "jquery-ui", "themes", "smoothness", "jquery-ui.min.css"), + pjoin(components, "marked", "lib", "marked.js"), + pjoin(components, "require", "require.js"), + pjoin(components, "underscore", "underscore-min.js"), + ]) + + # Ship all of Codemirror's CSS and JS + for parent, dirs, files in os.walk(pjoin(components, 'codemirror')): + for f in files: + if f.endswith(('.js', '.css')): + static_data.append(pjoin(parent, f)) + os.chdir(os.path.join('tests',)) js_tests = glob('casperjs/*.*') + glob('casperjs/*/*') From 4c41c331d7236c1a08e4a3321d689c51b8cf3d37 Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 31 Jan 2014 11:50:00 -0800 Subject: [PATCH 06/11] we only install man pages to usr/share let the function reflect this, rather than having several entries with no files. --- setupbase.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/setupbase.py b/setupbase.py index 0b8185619..38a41a173 100644 --- a/setupbase.py +++ b/setupbase.py @@ -236,10 +236,9 @@ def find_data_files(): """ Find IPython's data_files. - Most of these are docs. + Just man pages at this point. """ - docdirbase = pjoin('share', 'doc', 'ipython') manpagebase = pjoin('share', 'man', 'man1') # Simple file lists can be made by hand @@ -248,24 +247,8 @@ def find_data_files(): # When running from a source tree, the manpages aren't gzipped manpages = [f for f in glob(pjoin('docs','man','*.1')) if isfile(f)] - igridhelpfiles = [f for f in glob(pjoin('IPython','extensions','igrid_help.*')) if isfile(f)] - - # For nested structures, use the utility above - example_files = make_dir_struct( - 'data', - pjoin('docs','examples'), - pjoin(docdirbase,'examples') - ) - manual_files = make_dir_struct( - 'data', - pjoin('docs','html'), - pjoin(docdirbase,'manual') - ) - # And assemble the entire output list - data_files = [ (manpagebase, manpages), - (pjoin(docdirbase, 'extensions'), igridhelpfiles), - ] + manual_files + example_files + data_files = [ (manpagebase, manpages) ] return data_files From f1f56e714445f6e9ae6717004b54542e97a4ab3c Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 31 Jan 2014 12:20:31 -0800 Subject: [PATCH 07/11] only check for dependencies when installing without setuptools --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e74cfa128..615fe9cea 100755 --- a/setup.py +++ b/setup.py @@ -314,10 +314,13 @@ if 'setuptools' in sys.modules: "ipython_win_post_install.py"}} else: - # If we are running without setuptools, call this function which will + # If we are installing without setuptools, call this function which will # check for dependencies an inform the user what is needed. This is # just to make life easy for users. - check_for_dependencies() + for install_cmd in ('install', 'symlink'): + if install_cmd in sys.argv: + check_for_dependencies() + break # scripts has to be a non-empty list, or install_scripts isn't called setup_args['scripts'] = [e.split('=')[0].strip() for e in find_entry_points()] From 197dc25b97313ac10a0ed0dbec3c1b5e470ef66b Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 31 Jan 2014 14:42:02 -0800 Subject: [PATCH 08/11] fix missing-wheel command because an empty distutils command must implement three no-op methods and one class attribute. Which makes perfect sense. --- setupbase.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/setupbase.py b/setupbase.py index 38a41a173..83a8de8dd 100644 --- a/setupbase.py +++ b/setupbase.py @@ -570,8 +570,19 @@ def get_bdist_wheel(): Constructs py2-none-any tag, instead of py2.7-none-any """ class RequiresWheel(Command): + description = "Dummy command for missing bdist_wheel" + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + def run(self): print("bdist_wheel requires the wheel package") + sys.exit(1) + if 'setuptools' not in sys.modules: return RequiresWheel else: From 07f757789f96d55887d71f645ea03ce0b46764c8 Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 31 Jan 2014 14:42:40 -0800 Subject: [PATCH 09/11] validate package_data and fix errors caught by the validation --- setupbase.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/setupbase.py b/setupbase.py index 83a8de8dd..887150d53 100644 --- a/setupbase.py +++ b/setupbase.py @@ -149,13 +149,13 @@ def find_package_data(): static_data.extend([ pjoin(components, "backbone", "backbone-min.js"), pjoin(components, "bootstrap", "bootstrap", "js", "bootstrap.min.js"), - pjoin(components, "font-awesome", "build", "assets", "font", "*.*"), + pjoin(components, "font-awesome", "build", "assets", "font-awesome", "font", "*.*"), pjoin(components, "highlight.js", "build", "highlight.pack.js"), pjoin(components, "jquery", "jquery.min.js"), pjoin(components, "jquery-ui", "ui", "minified", "jquery-ui.min.js"), pjoin(components, "jquery-ui", "themes", "smoothness", "jquery-ui.min.css"), pjoin(components, "marked", "lib", "marked.js"), - pjoin(components, "require", "require.js"), + pjoin(components, "requirejs", "require.js"), pjoin(components, "underscore", "underscore-min.js"), ]) @@ -178,7 +178,6 @@ def find_package_data(): 'IPython.config.profile' : ['README*', '*/*.py'], 'IPython.core.tests' : ['*.png', '*.jpg'], 'IPython.lib.tests' : ['*.wav'], - 'IPython.testing' : ['*.txt'], 'IPython.testing.plugin' : ['*.txt'], 'IPython.html' : ['templates/*'] + static_data, 'IPython.html.tests' : js_tests, @@ -188,6 +187,17 @@ def find_package_data(): 'IPython.nbconvert.filters' : ['marked.js'], 'IPython.nbformat' : ['tests/*.ipynb'] } + + # verify that package_data makes sense + for pkg, data in package_data.items(): + pkg_root = pjoin(*pkg.split('.')) + for d in data: + path = pjoin(pkg_root, d) + if '*' in path: + assert len(glob(path)) > 0, "No files match pattern %s" % path + else: + assert os.path.exists(path), "Missing package data: %s" % path + return package_data From a2511de20df650104f3e9b02f4f4ad669b909e1e Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 3 Feb 2014 17:26:17 -0800 Subject: [PATCH 10/11] font-awesome moved --- setupbase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setupbase.py b/setupbase.py index 887150d53..c15e9abcf 100644 --- a/setupbase.py +++ b/setupbase.py @@ -149,7 +149,7 @@ def find_package_data(): static_data.extend([ pjoin(components, "backbone", "backbone-min.js"), pjoin(components, "bootstrap", "bootstrap", "js", "bootstrap.min.js"), - pjoin(components, "font-awesome", "build", "assets", "font-awesome", "font", "*.*"), + pjoin(components, "font-awesome", "font", "*.*"), pjoin(components, "highlight.js", "build", "highlight.pack.js"), pjoin(components, "jquery", "jquery.min.js"), pjoin(components, "jquery-ui", "ui", "minified", "jquery-ui.min.js"), From 0c62217720e57003b20a5b4bd165cd0dc836b9dc Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 3 Feb 2014 17:26:31 -0800 Subject: [PATCH 11/11] make all of the extras lists simpler, that way --- setup.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 615fe9cea..587a6e309 100755 --- a/setup.py +++ b/setup.py @@ -263,20 +263,17 @@ setuptools_extra_args = {} # setuptools requirements extras_require = dict( - parallel = 'pyzmq>=2.1.11', + parallel = ['pyzmq>=2.1.11'], qtconsole = ['pyzmq>=2.1.11', 'pygments'], - zmq = 'pyzmq>=2.1.11', + zmq = ['pyzmq>=2.1.11'], doc = ['Sphinx>=1.1', 'numpydoc'], - test = 'nose>=0.10.1', + test = ['nose>=0.10.1'], notebook = ['tornado>=3.1', 'pyzmq>=2.1.11', 'jinja2'], nbconvert = ['pygments', 'jinja2', 'Sphinx>=0.3'] ) everything = set() for deps in extras_require.values(): - if not isinstance(deps, list): - deps = [deps] - for dep in deps: - everything.add(dep) + everything.update(deps) extras_require['all'] = everything install_requires = [] if sys.platform == 'darwin': @@ -284,7 +281,6 @@ if sys.platform == 'darwin': elif sys.platform.startswith('win'): # Pyreadline has unicode and Python 3 fixes in 2.0 install_requires.append('pyreadline>=2.0') - if 'setuptools' in sys.modules: # setup.py develop should check for submodules