Perf work and Parallel builds

This commit is contained in:
Thomas Reynolds 2016-01-22 15:57:07 -08:00
parent b8c0fd34e7
commit c7a4618166
21 changed files with 63 additions and 43 deletions

View file

@ -19,6 +19,7 @@ master
* Fix asset_host in combination with Google Analytics snippet. #1751 * Fix asset_host in combination with Google Analytics snippet. #1751
* Show an error message when git CLI is not available. #1765 * Show an error message when git CLI is not available. #1765
* Correctly show file names of GZIP'ed assets. #1364 * Correctly show file names of GZIP'ed assets. #1364
* Build file output is not parallel-ized! Use `middleman build --no-parallel` to disable.
# 4.0.0 # 4.0.0

View file

@ -4,7 +4,7 @@ require 'middleman-core/profiling'
if ARGV.include? '--profile' if ARGV.include? '--profile'
Middleman::Profiling.profiler = Middleman::Profiling::RubyProfProfiler.new Middleman::Profiling.profiler = Middleman::Profiling::RubyProfProfiler.new
end end
Middleman::Profiling.start # Middleman::Profiling.start
require "middleman-core/load_paths" require "middleman-core/load_paths"
Middleman.setup_load_paths Middleman.setup_load_paths

View file

@ -14,6 +14,10 @@ module Middleman::Cli
type: :boolean, type: :boolean,
default: true, default: true,
desc: 'Remove orphaned files from build (--no-clean to disable)' desc: 'Remove orphaned files from build (--no-clean to disable)'
class_option :parallel,
type: :boolean,
default: true,
desc: 'Output files in parallel (--no-parallel to disable)'
class_option :glob, class_option :glob,
type: :string, type: :string,
aliases: '-g', aliases: '-g',

View file

@ -184,7 +184,7 @@ module Middleman
end end
}, 'Callbacks that can exclude paths from the sitemap' }, 'Callbacks that can exclude paths from the sitemap'
define_setting :skip_build_clean, proc { |p| [/\.git/].any? { |r| r.match(p) } }, 'Whether some paths should not be removed during a clean build.' define_setting :skip_build_clean, proc { |p| [/\.git/].any? { |r| p =~ r } }, 'Whether some paths should not be removed during a clean build.'
define_setting :watcher_disable, false, 'If the Listen watcher should not run' define_setting :watcher_disable, false, 'If the Listen watcher should not run'
define_setting :watcher_force_polling, false, 'If the Listen watcher should run in polling mode' define_setting :watcher_force_polling, false, 'If the Listen watcher should run in polling mode'
@ -314,7 +314,7 @@ module Middleman
# Clean up missing Tilt exts # Clean up missing Tilt exts
def prune_tilt_templates! def prune_tilt_templates!
::Tilt.mappings.each do |key, _| ::Tilt.mappings.each_key do |key|
begin begin
::Tilt[".#{key}"] ::Tilt[".#{key}"]
rescue LoadError, NameError rescue LoadError, NameError

View file

@ -1,6 +1,7 @@
require 'pathname' require 'pathname'
require 'fileutils' require 'fileutils'
require 'tempfile' require 'tempfile'
require 'parallel'
require 'middleman-core/rack' require 'middleman-core/rack'
require 'middleman-core/callback_manager' require 'middleman-core/callback_manager'
require 'middleman-core/contracts' require 'middleman-core/contracts'
@ -36,6 +37,7 @@ module Middleman
@glob = opts.fetch(:glob) @glob = opts.fetch(:glob)
@cleaning = opts.fetch(:clean) @cleaning = opts.fetch(:clean)
@parallel = opts.fetch(:parallel, true)
rack_app = ::Middleman::Rack.new(@app).to_app rack_app = ::Middleman::Rack.new(@app).to_app
@rack = ::Rack::MockRequest.new(rack_app) @rack = ::Rack::MockRequest.new(rack_app)
@ -63,16 +65,18 @@ module Middleman
prerender_css prerender_css
end end
::Middleman::Profiling.start
::Middleman::Util.instrument 'builder.output' do ::Middleman::Util.instrument 'builder.output' do
output_files output_files
end end
::Middleman::Profiling.report('build')
::Middleman::Util.instrument 'builder.clean' do ::Middleman::Util.instrument 'builder.clean' do
clean! if @cleaning clean! if @cleaning
end end
::Middleman::Profiling.report('build')
::Middleman::Util.instrument 'builder.after' do ::Middleman::Util.instrument 'builder.after' do
@app.execute_callbacks(:after_build, [self]) @app.execute_callbacks(:after_build, [self])
end end
@ -87,9 +91,8 @@ module Middleman
logger.debug '== Prerendering CSS' logger.debug '== Prerendering CSS'
css_files = ::Middleman::Util.instrument 'builder.prerender.output' do css_files = ::Middleman::Util.instrument 'builder.prerender.output' do
@app.sitemap.resources resources = @app.sitemap.resources.select { |resource| resource.ext == '.css' }
.select { |resource| resource.ext == '.css' } output_resources(resources)
.each(&method(:output_resource))
end end
::Middleman::Util.instrument 'builder.prerender.check-files' do ::Middleman::Util.instrument 'builder.prerender.check-files' do
@ -117,7 +120,20 @@ module Middleman
resources = resources.select { |resource| File.fnmatch(@glob, resource.destination_path) } resources = resources.select { |resource| File.fnmatch(@glob, resource.destination_path) }
end end
resources.each(&method(:output_resource)) output_resources(resources)
end
Contract ResourceList => ResourceList
def output_resources(resources)
cleaned_paths = if @parallel
::Parallel.map(resources, &method(:output_resource))
else
resources.map(&method(:output_resource))
end
cleaned_paths.each { |p| @to_clean.delete(p) } if @cleaning
resources
end end
# Figure out the correct event mode. # Figure out the correct event mode.
@ -181,7 +197,7 @@ module Middleman
# Try to output a resource and capture errors. # Try to output a resource and capture errors.
# @param [Middleman::Sitemap::Resource] resource The resource. # @param [Middleman::Sitemap::Resource] resource The resource.
# @return [void] # @return [void]
Contract IsA['Middleman::Sitemap::Resource'] => Any Contract IsA['Middleman::Sitemap::Resource'] => Maybe[Pathname]
def output_resource(resource) def output_resource(resource)
output_file = nil output_file = nil
@ -218,7 +234,7 @@ module Middleman
output_file output_file
end end
@to_clean.delete(Pathname(cleaned_name)) Pathname(cleaned_name)
end end
# Get a list of all the paths in the destination folder and save them # Get a list of all the paths in the destination folder and save them

View file

@ -203,11 +203,11 @@ module Middleman
def to_h def to_h
data = {} data = {}
store.each do |k, _| store.each_key do |k|
data[k] = data_for_path(k) data[k] = data_for_path(k)
end end
callbacks.each do |k, _| callbacks.each_key do |k|
data[k] = data_for_path(k) data[k] = data_for_path(k)
end end

View file

@ -113,7 +113,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
path_options = {} path_options = {}
path_options[:relative] = options.delete(:relative) if options.key?(:relative) path_options[:relative] = options.delete(:relative) if options.key?(:relative)
sources.flatten.inject(::ActiveSupport::SafeBuffer.new) do |all, source| sources.flatten.reduce(::ActiveSupport::SafeBuffer.new) do |all, source|
all << tag(:link, { all << tag(:link, {
href: asset_path(:css, source, path_options) href: asset_path(:css, source, path_options)
}.update(options)) }.update(options))
@ -127,7 +127,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
path_options = {} path_options = {}
path_options[:relative] = options.delete(:relative) if options.key?(:relative) path_options[:relative] = options.delete(:relative) if options.key?(:relative)
sources.flatten.inject(::ActiveSupport::SafeBuffer.new) do |all, source| sources.flatten.reduce(::ActiveSupport::SafeBuffer.new) do |all, source|
all << content_tag(:script, nil, { all << content_tag(:script, nil, {
src: asset_path(:js, source, path_options) src: asset_path(:js, source, path_options)
}.update(options)) }.update(options))
@ -152,7 +152,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
# If the basename of the request as no extension, assume we are serving a # If the basename of the request as no extension, assume we are serving a
# directory and join index_file to the path. # directory and join index_file to the path.
path = File.join(asset_dir, current_resource.path) path = File.join(asset_dir, current_resource.path)
path = path.sub(/#{Regexp.escape(File.extname(path))}$/, ".#{asset_ext}") path = path[0..-(File.extname(path).length + 1)] + ".#{asset_ext}"
yield path if sitemap.find_resource_by_path(path) yield path if sitemap.find_resource_by_path(path)
end end
@ -191,7 +191,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
# @param [Hash] options Data to pass through. # @param [Hash] options Data to pass through.
# @return [String] # @return [String]
def asset_path(kind, source, options={}) def asset_path(kind, source, options={})
options_with_resource = options.merge(current_resource: current_resource) options_with_resource = {}.merge!(options).merge!(current_resource: current_resource)
::Middleman::Util.asset_path(app, kind, source, options_with_resource) ::Middleman::Util.asset_path(app, kind, source, options_with_resource)
end end
@ -202,7 +202,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
# @param [Hash] options Additional options. # @param [Hash] options Additional options.
# @return [String] The fully qualified asset url # @return [String] The fully qualified asset url
def asset_url(path, prefix='', options={}) def asset_url(path, prefix='', options={})
options_with_resource = options.merge(current_resource: current_resource) options_with_resource = {}.merge!(options).merge!(current_resource: current_resource)
::Middleman::Util.asset_url(app, path, prefix, options_with_resource) ::Middleman::Util.asset_url(app, path, prefix, options_with_resource)
end end
@ -210,7 +210,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
# or a Resource, this will produce the nice URL configured for that # or a Resource, this will produce the nice URL configured for that
# path, respecting :relative_links, directory indexes, etc. # path, respecting :relative_links, directory indexes, etc.
def url_for(path_or_resource, options={}) def url_for(path_or_resource, options={})
options_with_resource = options.merge(current_resource: current_resource) options_with_resource = {}.merge!(options).merge!(current_resource: current_resource)
::Middleman::Util.url_for(app, path_or_resource, options_with_resource) ::Middleman::Util.url_for(app, path_or_resource, options_with_resource)
end end

View file

@ -203,7 +203,7 @@ class Middleman::CoreExtensions::Internationalization < ::Middleman::Extension
if (options[:mount_at_root] == locale) || (options[:mount_at_root].nil? && locales[0] == locale) if (options[:mount_at_root] == locale) || (options[:mount_at_root].nil? && locales[0] == locale)
'/' '/'
else else
replacement = options[:locale_map].fetch(locale, locale) replacement = options[:locale_map][locale] || locale
options[:path].sub(':locale', replacement.to_s).sub(':lang', replacement.to_s) # Backward compat options[:path].sub(':locale', replacement.to_s).sub(':lang', replacement.to_s) # Backward compat
end end
end end

View file

@ -108,7 +108,7 @@ module Middleman
ignore = rewriter.fetch(:ignore) ignore = rewriter.fetch(:ignore)
next if ignore.any? { |r| should_ignore?(r, full_asset_path) } next if ignore.any? { |r| should_ignore?(r, full_asset_path) }
rewrite_ignore = Array(rewriter.fetch(:rewrite_ignore, [])) rewrite_ignore = Array(rewriter[:rewrite_ignore] || [])
next if rewrite_ignore.any? { |i| ::Middleman::Util.path_match(i, path) } next if rewrite_ignore.any? { |i| ::Middleman::Util.path_match(i, path) }
proc = rewriter.fetch(:proc) proc = rewriter.fetch(:proc)
@ -132,7 +132,7 @@ module Middleman
def should_ignore?(validator, value) def should_ignore?(validator, value)
if validator.is_a? Regexp if validator.is_a? Regexp
# Treat as Regexp # Treat as Regexp
!value.match(validator).nil? !!(value =~ validator)
elsif validator.respond_to? :call elsif validator.respond_to? :call
# Treat as proc # Treat as proc
validator.call(value) validator.call(value)

View file

@ -432,7 +432,7 @@ module Middleman
{} {}
end end
sum.merge(resource_definitions) sum.merge!(resource_definitions)
end end
resources + generator_defs.map do |path, g| resources + generator_defs.map do |path, g|

View file

@ -106,7 +106,7 @@ module Middleman
# A flattened list of all extensions which are automatically activated # A flattened list of all extensions which are automatically activated
# @return [Array<Symbol>] A list of extension names which are automatically activated. # @return [Array<Symbol>] A list of extension names which are automatically activated.
def auto_activated def auto_activated
@auto_activate.values.map(&:to_a).flatten.map(&:name) @auto_activate.values.map(&:to_a).flat_map(&:name)
end end
# @api private # @api private

View file

@ -19,7 +19,7 @@ class Middleman::Extensions::AutomaticImageSizes < ::Middleman::Extension
real_path = path.dup real_path = path.dup
real_path = File.join(config[:images_dir], real_path) unless real_path.start_with?('/') real_path = File.join(config[:images_dir], real_path) unless real_path.start_with?('/')
file = app.files.find(:source, real_path) || app.files.find(:source, real_path.gsub(/^\//, '')) file = app.files.find(:source, real_path) || app.files.find(:source, real_path.sub(/^\//, ''))
if file && file[:full_path].exist? if file && file[:full_path].exist?
begin begin

View file

@ -23,7 +23,7 @@ class Middleman::Extensions::MinifyCss < ::Middleman::Extension
class SassCompressor class SassCompressor
def self.compress(style, options={}) def self.compress(style, options={})
root_node = ::Sass::SCSS::CssParser.new(style, 'middleman-css-input', 1).parse root_node = ::Sass::SCSS::CssParser.new(style, 'middleman-css-input', 1).parse
root_node.options = options.merge(style: :compressed) root_node.options = {}.merge!(options).merge!(style: :compressed)
root_node.render.strip root_node.render.strip
end end
end end

View file

@ -51,7 +51,7 @@ module Middleman
# Merge per-extension options from config # Merge per-extension options from config
extension = File.extname(path) extension = File.extname(path)
options = opts.merge(options_for_ext(extension)) options = {}.merge!(opts).merge!(options_for_ext(extension))
options[:outvar] ||= '@_out_buf' options[:outvar] ||= '@_out_buf'
options[:context] = context options[:context] = context
options.delete(:layout) options.delete(:layout)

View file

@ -30,7 +30,7 @@ module Middleman
end end
def evaluate(scope, locals, &block) def evaluate(scope, locals, &block)
options = @options.merge(filename: eval_file, line: line, context: @context || scope) options = {}.merge!(@options).merge!(filename: eval_file, line: line, context: @context || scope)
@engine = ::Haml::Engine.new(data, options) @engine = ::Haml::Engine.new(data, options)
output = @engine.render(scope, locals, &block) output = @engine.render(scope, locals, &block)

View file

@ -26,7 +26,7 @@ module Middleman
if ::Less.const_defined? :Engine if ::Less.const_defined? :Engine
@engine = ::Less::Engine.new(data) @engine = ::Less::Engine.new(data)
else else
parser = ::Less::Parser.new(options.merge(filename: eval_file, line: line, paths: ['.', File.dirname(eval_file)])) parser = ::Less::Parser.new({}.merge!(options).merge!(filename: eval_file, line: line, paths: ['.', File.dirname(eval_file)]))
@engine = parser.parse(data) @engine = parser.parse(data)
end end
end end

View file

@ -81,7 +81,7 @@ module Middleman
filename: eval_file, filename: eval_file,
line: line, line: line,
syntax: syntax, syntax: syntax,
custom: (options[:custom] || {}).merge( custom: {}.merge!(options[:custom] || {}).merge!(
middleman_context: ctx.app, middleman_context: ctx.app,
current_resource: ctx.current_resource current_resource: ctx.current_resource
) )
@ -97,7 +97,7 @@ module Middleman
more_opts[:css_filename] = file.sub(/\.s[ac]ss$/, '') more_opts[:css_filename] = file.sub(/\.s[ac]ss$/, '')
end end
options.merge(more_opts) {}.merge!(options).merge!(more_opts)
end end
end end

View file

@ -189,6 +189,7 @@ module Middleman
array_of_types = Array(types) array_of_types = Array(types)
watchers watchers
.lazy
.select { |d| array_of_types.include?(d.type) } .select { |d| array_of_types.include?(d.type) }
.map { |d| d.find(path, glob) } .map { |d| d.find(path, glob) }
.reject(&:nil?) .reject(&:nil?)
@ -202,9 +203,7 @@ module Middleman
# @return [Boolean] # @return [Boolean]
Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String => Bool Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String => Bool
def exists?(types, path) def exists?(types, path)
watchers watchers.any? { |d| Array(types).include?(d.type) && d.exists?(path) }
.select { |d| Array(types).include?(d.type) }
.any? { |d| d.exists?(path) }
end end
# Check if a file for a given type exists. # Check if a file for a given type exists.
@ -214,9 +213,7 @@ module Middleman
# @return [Boolean] # @return [Boolean]
Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String => Maybe[HANDLER] Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String => Maybe[HANDLER]
def watcher_for_path(types, path) def watcher_for_path(types, path)
watchers watchers.detect { |d| Array(types).include?(d.type) && d.exists?(path) }
.select { |d| Array(types).include?(d.type) }
.find { |d| d.exists?(path) }
end end
# Manually check for new files # Manually check for new files
@ -317,7 +314,7 @@ module Middleman
def matches?(validator, file) def matches?(validator, file)
path = file[:relative_path] path = file[:relative_path]
if validator.is_a? Regexp if validator.is_a? Regexp
!!validator.match(path.to_s) !!(path.to_s =~ validator)
else else
!!validator.call(path, @app) !!validator.call(path, @app)
end end

View file

@ -322,7 +322,7 @@ module Middleman
if @only.empty? if @only.empty?
!@ignored.call(file) !@ignored.call(file)
else else
@only.any? { |reg| reg.match(file[:relative_path].to_s) } @only.any? { |reg| file[:relative_path].to_s =~ reg }
end end
end end
end end

View file

@ -79,7 +79,7 @@ module Middleman
path == matcher path == matcher
end end
when matcher.respond_to?(:match) when matcher.respond_to?(:match)
!matcher.match(path).nil? !!(path =~ matcher)
when matcher.respond_to?(:call) when matcher.respond_to?(:call)
matcher.call(path) matcher.call(path)
else else
@ -357,7 +357,7 @@ module Middleman
begin begin
uri = ::Addressable::URI.parse(asset_path) uri = ::Addressable::URI.parse(asset_path)
if uri.relative? && uri.host.nil? && !asset_path.match(/^[^\/].*[a-z]+\.[a-z]+\/.*/) && (result = yield(asset_path)) if uri.relative? && uri.host.nil? && !(asset_path =~ /^[^\/].*[a-z]+\.[a-z]+\/.*/) && (result = yield(asset_path))
"#{opening_character}#{result}" "#{opening_character}#{result}"
else else
match match
@ -456,10 +456,11 @@ module Middleman
Contract String => String Contract String => String
def step_through_extensions(path) def step_through_extensions(path)
while ::Tilt[path] while ::Tilt[path]
yield File.extname(path) if block_given? ext = File.extname(path)
yield ext if block_given?
# Strip templating extensions as long as Tilt knows them # Strip templating extensions as long as Tilt knows them
path = path.sub(/#{::Regexp.escape(File.extname(path))}$/, '') path = path[0..-(ext.length + 1)]
end end
yield File.extname(path) if block_given? yield File.extname(path) if block_given?

View file

@ -24,6 +24,7 @@ Gem::Specification.new do |s|
s.add_dependency('tilt', ['~> 1.4.1']) s.add_dependency('tilt', ['~> 1.4.1'])
s.add_dependency('erubis') s.add_dependency('erubis')
s.add_dependency('fast_blank') s.add_dependency('fast_blank')
s.add_dependency('parallel')
# Helpers # Helpers
s.add_dependency('activesupport', ['~> 4.2']) s.add_dependency('activesupport', ['~> 4.2'])