From e00931bc110107690f58adabd0fa3c220ec35731 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 29 Apr 2015 14:04:59 -0700 Subject: [PATCH 1/8] compile main.min.js for each application - build-main.js is a separate file, spawned with process.fork to allow parallel builds - gulp js builds main.min.js for each js application --- .gitignore | 1 + build-main.js | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++ gulpfile.js | 46 +++++++++++++++++++++++++++++++++++++++-- package.json | 1 + 4 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 build-main.js diff --git a/.gitignore b/.gitignore index d10a9932c..d142057e4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ docs/source/api/generated docs/gh-pages notebook/static/components notebook/static/style/*.min.css* +notebook/static/*/js/main.min.js* node_modules *.py[co] __pycache__ diff --git a/build-main.js b/build-main.js new file mode 100644 index 000000000..5936b791d --- /dev/null +++ b/build-main.js @@ -0,0 +1,57 @@ +// build main.min.js +// spawned by gulp to allow parallelism + +var rjs = require('requirejs').optimize; + +var name = process.argv[2]; + +var rjs_config = { + name: name + '/js/main', + out: './notebook/static/' + name + '/js/main.min.js', + baseUrl: 'notebook/static', + preserveLicenseComments: false, // license comments conflict with sourcemap generation + generateSourceMaps: true, + optimize: "uglify2", + paths: { + underscore : 'components/underscore/underscore-min', + backbone : 'components/backbone/backbone-min', + jquery: 'components/jquery/jquery.min', + bootstrap: 'components/bootstrap/js/bootstrap.min', + bootstraptour: 'components/bootstrap-tour/build/js/bootstrap-tour.min', + jqueryui: 'components/jquery-ui/ui/minified/jquery-ui.min', + moment: 'components/moment/moment', + codemirror: 'components/codemirror', + termjs: 'components/term.js/src/term', + contents: 'empty:' + }, + shim: { + underscore: { + exports: '_' + }, + backbone: { + deps: ["underscore", "jquery"], + exports: "Backbone" + }, + bootstrap: { + deps: ["jquery"], + exports: "bootstrap" + }, + bootstraptour: { + deps: ["bootstrap"], + exports: "Tour" + }, + jqueryui: { + deps: ["jquery"], + exports: "$" + } + }, + + exclude: [ + "custom/custom", + ] +}; + +rjs(rjs_config, console.log, function (err) { + console.log("Failed to build", name, err); + process.exit(1); +}); diff --git a/gulpfile.js b/gulpfile.js index 3bb0297be..2e9b4992c 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -1,13 +1,15 @@ +var fork = require('child_process').fork; +var path = require('path'); + var gulp = require('gulp'); var less = require('gulp-less'); -var path = require('path'); var minifyCSS = require('gulp-minify-css'); var rename = require('gulp-rename'); var sourcemaps = require('gulp-sourcemaps'); // now some dev nice utilities. var livereload = require('gulp-livereload'); - + gulp.task('css', function () { return gulp.src('./notebook/static/style/*.less') .pipe(sourcemaps.init()) @@ -23,9 +25,49 @@ gulp.task('css', function () { .pipe(livereload()); }); +function build_main(name, callback) { + // build main.min.js for a given application name + // run in a subprocess to allow parallel builds + // clone requirejs config + var p = fork('./build-main.js', [name]); + p.on('exit', function (code, status) { + if (code) { + callback(new Error("Build failed")); + } else { + callback(); + } + }); + return; +} +// build notebook-js, edit-js, etc. tasks +// which enables parallelism +var apps = ['edit', 'notebook', 'terminal', 'tree']; + +apps.map(function (name) { + gulp.task(name + '-js', function (finish) { + build_main(name, finish); + }); +}); + +gulp.task('js', apps.map(function (name) { return name + '-js'; })); gulp.task('watch', function() { livereload.listen(); gulp.watch('notebook/static/**/*.less', ['css']); + + var s = path.join('notebook', 'static'); + + function alljs(name) { + return path.join(s, name, '**', '*.js'); + } + var common_js = ['components', 'base', 'auth', 'services'].map(alljs); + + gulp.watch(common_js, ['js']); + apps.map(function (name) { + gulp.watch([ + alljs(name), + '!' + path.join(s, name, 'js', 'main.min.js'), + ], [name + '-js']); + }); }); diff --git a/package.json b/package.json index 93f03877d..381a854cc 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "gulp-rename": "^1.2.2", "gulp-sourcemaps": "^1.5.1", "less": "~2", + "requirejs": "^2.1.17", "source-map": "^0.4.2" } } From 85d952b4d75de93fdb9bba9c36ae971b174a0d41 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 29 Apr 2015 15:35:29 -0700 Subject: [PATCH 2/8] use minified `main.min.js` on each application --- notebook/templates/edit.html | 2 +- notebook/templates/notebook.html | 2 +- notebook/templates/terminal.html | 2 +- notebook/templates/tree.html | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/notebook/templates/edit.html b/notebook/templates/edit.html index 72d748169..1c543fc2f 100644 --- a/notebook/templates/edit.html +++ b/notebook/templates/edit.html @@ -95,5 +95,5 @@ data-file-path="{{file_path}}" {{super()}} - + {% endblock %} diff --git a/notebook/templates/notebook.html b/notebook/templates/notebook.html index c065d8cb4..9661fe272 100644 --- a/notebook/templates/notebook.html +++ b/notebook/templates/notebook.html @@ -324,6 +324,6 @@ data-notebook-path="{{notebook_path}}" - + {% endblock %} diff --git a/notebook/templates/terminal.html b/notebook/templates/terminal.html index ca14f4619..e174b55e4 100644 --- a/notebook/templates/terminal.html +++ b/notebook/templates/terminal.html @@ -60,5 +60,5 @@ data-ws-path="{{ws_path}}" {{super()}} - + {% endblock %} diff --git a/notebook/templates/tree.html b/notebook/templates/tree.html index d2b9f8686..cd15f702b 100644 --- a/notebook/templates/tree.html +++ b/notebook/templates/tree.html @@ -183,5 +183,5 @@ data-terminals-available="{{terminals_available}}" {% block script %} {{super()}} - + {% endblock %} From 0deee0d9893685598e0b9ef4d9aa880bc923a9c3 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 4 May 2015 13:33:39 -0700 Subject: [PATCH 3/8] build auth/js/main.min.js exports login_main and logout_main --- gulpfile.js | 2 +- notebook/static/auth/js/loginmain.js | 16 +++++++++------- notebook/static/auth/js/logoutmain.js | 12 +++++++----- notebook/static/auth/js/main.js | 9 +++++++++ notebook/templates/login.html | 6 +++++- notebook/templates/logout.html | 6 +++++- notebook/templates/page.html | 1 + 7 files changed, 37 insertions(+), 15 deletions(-) create mode 100644 notebook/static/auth/js/main.js diff --git a/gulpfile.js b/gulpfile.js index 2e9b4992c..5760b4fb9 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -42,7 +42,7 @@ function build_main(name, callback) { // build notebook-js, edit-js, etc. tasks // which enables parallelism -var apps = ['edit', 'notebook', 'terminal', 'tree']; +var apps = ['edit', 'notebook', 'terminal', 'tree', 'auth']; apps.map(function (name) { gulp.task(name + '-js', function (finish) { diff --git a/notebook/static/auth/js/loginmain.js b/notebook/static/auth/js/loginmain.js index 44a2bf31d..1e312ee69 100644 --- a/notebook/static/auth/js/loginmain.js +++ b/notebook/static/auth/js/loginmain.js @@ -1,12 +1,14 @@ // Copyright (c) Jupyter Development Team. // Distributed under the terms of the Modified BSD License. -var ipython = ipython || {}; -require(['base/js/page'], function(page) { - var page_instance = new page.Page(); - $('button#login_submit').addClass("btn btn-default"); - page_instance.show(); - $('input#password_input').focus(); +define(['base/js/namespace', 'base/js/page'], function(IPython, page) { + function login_main() { + var page_instance = new page.Page(); + $('button#login_submit').addClass("btn btn-default"); + page_instance.show(); + $('input#password_input').focus(); - ipython.page = page_instance; + IPython.page = page_instance; + } + return login_main; }); diff --git a/notebook/static/auth/js/logoutmain.js b/notebook/static/auth/js/logoutmain.js index b91b0fc4b..7b3f6b4da 100644 --- a/notebook/static/auth/js/logoutmain.js +++ b/notebook/static/auth/js/logoutmain.js @@ -1,10 +1,12 @@ // Copyright (c) Jupyter Development Team. // Distributed under the terms of the Modified BSD License. -var ipython = ipython || {}; -require(['base/js/page'], function(page) { - var page_instance = new page.Page(); - page_instance.show(); +define(['base/js/namespace', 'base/js/page'], function(IPython, page) { + function logout_main() { + var page_instance = new page.Page(); + page_instance.show(); - ipython.page = page_instance; + IPython.page = page_instance; + } + return logout_main; }); diff --git a/notebook/static/auth/js/main.js b/notebook/static/auth/js/main.js new file mode 100644 index 000000000..7be82388e --- /dev/null +++ b/notebook/static/auth/js/main.js @@ -0,0 +1,9 @@ +// Copyright (c) Jupyter Development Team. +// Distributed under the terms of the Modified BSD License. + +define(['./loginmain', './logoutmain'], function (login_main, logout_main) { + return { + login_main: login_main, + logout_main: logout_main + }; +}); diff --git a/notebook/templates/login.html b/notebook/templates/login.html index 54223360d..78a5d6645 100644 --- a/notebook/templates/login.html +++ b/notebook/templates/login.html @@ -48,6 +48,10 @@ {% block script %} {{super()}} - + {% endblock %} diff --git a/notebook/templates/logout.html b/notebook/templates/logout.html index 91595aacb..81c3c9118 100644 --- a/notebook/templates/logout.html +++ b/notebook/templates/logout.html @@ -34,6 +34,10 @@ {% block script %} {{super()}} - + {% endblock %} diff --git a/notebook/templates/page.html b/notebook/templates/page.html index 3baea5910..6b1939dff 100644 --- a/notebook/templates/page.html +++ b/notebook/templates/page.html @@ -23,6 +23,7 @@ {% endif %} baseUrl: '{{static_url("", include_version=False)}}', paths: { + 'auth/js/main': 'auth/js/main.min', custom : '{{ base_url }}custom', nbextensions : '{{ base_url }}nbextensions', widgets : '{{ base_url }}deprecatedwidgets', From a58e885e2cc182a4a952696a8097bda8ff811270 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 4 May 2015 13:35:02 -0700 Subject: [PATCH 4/8] reorder js minification based on duration start slowest first --- gulpfile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gulpfile.js b/gulpfile.js index 5760b4fb9..5b0c18b62 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -42,7 +42,7 @@ function build_main(name, callback) { // build notebook-js, edit-js, etc. tasks // which enables parallelism -var apps = ['edit', 'notebook', 'terminal', 'tree', 'auth']; +var apps = ['notebook', 'tree', 'edit', 'terminal', 'auth']; apps.map(function (name) { gulp.task(name + '-js', function (finish) { From d54d7904ee6d4447284d91aad66300889e8935ee Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 4 May 2015 15:51:56 -0700 Subject: [PATCH 5/8] trigger `gulp js` as distutils step ensures minified js is installed --- setup.py | 6 ++++-- setupbase.py | 48 +++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index a10b30976..f1310f3d6 100755 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ from setupbase import ( find_package_data, check_package_data_first, CompileCSS, + CompileJS, Bower, JavascriptVersion, css_js_prerelease, @@ -107,9 +108,10 @@ from distutils.command.sdist import sdist setup_args['cmdclass'] = { 'build_py': css_js_prerelease( check_package_data_first(build_py)), - 'sdist' : css_js_prerelease(sdist), + 'sdist' : css_js_prerelease(sdist, strict=True), 'css' : CompileCSS, - 'js' : Bower, + 'js' : CompileJS, + 'jsdeps' : Bower, 'jsversion' : JavascriptVersion, } diff --git a/setupbase.py b/setupbase.py index b42944cea..0d1724fa7 100644 --- a/setupbase.py +++ b/setupbase.py @@ -361,14 +361,41 @@ class CompileCSS(Command): pass def run(self): - - self.run_command('js') + self.run_command('jsdeps') env = os.environ.copy() env['PATH'] = npm_path try: check_call(['gulp','css'], cwd=repo_root, env=env) except OSError as e: - print("Failed to run gulp: %s" % e, file=sys.stderr) + print("Failed to run gulp css: %s" % e, file=sys.stderr) + print("You can install js dependencies with `npm install`", file=sys.stderr) + raise + # update package data in case this created new files + update_package_data(self.distribution) + + +class CompileJS(Command): + """Rebuild minified Notebook Javascript + + Calls `gulp js` + """ + description = "Rebuild Notebook Javascript" + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + self.run_command('jsdeps') + env = os.environ.copy() + env['PATH'] = npm_path + try: + check_call(['gulp','js'], cwd=repo_root, env=env) + except OSError as e: + print("Failed to run gulp js: %s" % e, file=sys.stderr) print("You can install js dependencies with `npm install`", file=sys.stderr) raise # update package data in case this created new files @@ -401,19 +428,26 @@ class JavascriptVersion(Command): raise RuntimeError("Didn't find IPython.version line in %s" % nsfile) -def css_js_prerelease(command): - """decorator for building js/minified css prior to another command""" +def css_js_prerelease(command, strict=False): + """decorator for building minified js/css prior to another command""" class DecoratedCommand(command): def run(self): self.distribution.run_command('jsversion') + jsdeps = self.distribution.get_command_obj('jsdeps') + jsdeps.force = True js = self.distribution.get_command_obj('js') js.force = True css = self.distribution.get_command_obj('css') css.force = True try: self.distribution.run_command('css') + self.distribution.run_command('js') except Exception as e: - log.warn("rebuilding css and sourcemaps failed (not a problem)") - log.warn(str(e)) + if strict: + log.warn("rebuilding js and css failed") + raise e + else: + log.warn("rebuilding js and css failed (not a problem)") + log.warn(str(e)) command.run(self) return DecoratedCommand From 3cf959734d883735bb9260d94a6613e22bb6aeaf Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 4 May 2015 15:58:15 -0700 Subject: [PATCH 6/8] prevent installation if main.min.js are missing --- setupbase.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setupbase.py b/setupbase.py index 0d1724fa7..f349900be 100644 --- a/setupbase.py +++ b/setupbase.py @@ -107,7 +107,12 @@ def find_package_data(): continue for f in files: static_data.append(pjoin(parent, f)) - + + # for verification purposes, explicitly add main.min.js + # so that installation will fail if they are missing + for app in ['auth', 'edit', 'notebook', 'terminal', 'tree']: + static_data.append(pjoin('static', app, 'js', 'main.min.js')) + components = pjoin("static", "components") # select the components we actually need to install # (there are lots of resources we bundle for sdist-reasons that we don't actually use) From 30102f278072179225244a742c7296c995593600 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 4 May 2015 19:43:07 -0700 Subject: [PATCH 7/8] use gulp-newer to avoid wasteful re-running of rjs --- gulpfile.js | 41 ++++++++++++++++++++++++++++++++++++++++- package.json | 5 ++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/gulpfile.js b/gulpfile.js index 5b0c18b62..585b18412 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -1,9 +1,13 @@ var fork = require('child_process').fork; +var fs = require('fs'); var path = require('path'); +var File = require('vinyl'); +var through = require('through'); var gulp = require('gulp'); var less = require('gulp-less'); var minifyCSS = require('gulp-minify-css'); +var newer = require('gulp-newer'); var rename = require('gulp-rename'); var sourcemaps = require('gulp-sourcemaps'); @@ -46,7 +50,42 @@ var apps = ['notebook', 'tree', 'edit', 'terminal', 'auth']; apps.map(function (name) { gulp.task(name + '-js', function (finish) { - build_main(name, finish); + var s = path.join('notebook', 'static'); + var src = path.join(s, name, 'js', 'main.js'); + var rel_dest = path.join(name, 'js', 'main.min.js'); + var dest = path.join(s, rel_dest); + + var sources = [ + path.join(s, name, 'js', '*.js'), + path.join(s, "base", 'js', '*.js'), + path.join(s, "auth", 'js', '*.js'), + path.join(s, "services", 'config.js'), + ]; + + // for required_components + if (name === 'notebook') { + sources.push(path.join(s, "services", '**', '*.js')); + } + + fs.readdirSync(path.join(s, 'components')).map(function (c) { + if (c !== 'MathJax') { + // skip MathJax because it has tons of files and makes everything super slow + sources.push(path.join(s, 'components', c, '**', '*.js')); + } + }); + + gulp.src(sources, {base: s}) + .pipe(newer(dest)) + .pipe(through(function(file) { + console.log(file.relative + " has changed, updating " + rel_dest); + this.pause(); + build_main(name, finish); + return; + })) + .on('end', function () { + console.log(rel_dest + " up to date"); + finish(); + }); }); }); diff --git a/package.json b/package.json index 381a854cc..4878fc326 100644 --- a/package.json +++ b/package.json @@ -14,10 +14,13 @@ "gulp-less": "^3.0.2", "gulp-livereload": "^3.8.0", "gulp-minify-css": "^1.0.0", + "gulp-newer": "^0.5.0", "gulp-rename": "^1.2.2", "gulp-sourcemaps": "^1.5.1", "less": "~2", "requirejs": "^2.1.17", - "source-map": "^0.4.2" + "source-map": "^0.4.2", + "through": "^2.3.7", + "vinyl": "^0.4.6" } } From 7923bc817cfc8cde8d5f1162472afe09fd76eef3 Mon Sep 17 00:00:00 2001 From: Min RK Date: Sat, 23 May 2015 14:44:25 -0700 Subject: [PATCH 8/8] Clarify dependency-changed message and add a few clarifying comments --- gulpfile.js | 7 +++++-- package.json | 3 +-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/gulpfile.js b/gulpfile.js index 585b18412..dd3ad6fec 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -2,7 +2,6 @@ var fork = require('child_process').fork; var fs = require('fs'); var path = require('path'); -var File = require('vinyl'); var through = require('through'); var gulp = require('gulp'); var less = require('gulp-less'); @@ -74,15 +73,19 @@ apps.map(function (name) { } }); + // sources is a greedy list, containing all dependencies plus some for simplicity. gulp.src(sources, {base: s}) .pipe(newer(dest)) .pipe(through(function(file) { - console.log(file.relative + " has changed, updating " + rel_dest); + // if any dependency changed, update main.min.js + console.log("A dependency has changed, updating " + rel_dest); + // pause halts the pipeline this.pause(); build_main(name, finish); return; })) .on('end', function () { + // if we get here, no dependency is newer than the target console.log(rel_dest + " up to date"); finish(); }); diff --git a/package.json b/package.json index 4878fc326..d5cf1ac7e 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,6 @@ "less": "~2", "requirejs": "^2.1.17", "source-map": "^0.4.2", - "through": "^2.3.7", - "vinyl": "^0.4.6" + "through": "^2.3.7" } }