deac/middleman
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Perf work and Parallel builds

Browse source
This commit is contained in:
Thomas Reynolds 2016-01-22 15:57:07 -08:00
parent b8c0fd34e7
commit c7a4618166
21 changed files with 63 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -19,6 +19,7 @@ master
* Fix asset_host in combination with Google Analytics snippet. #1751
* Show an error message when git CLI is not available. #1765
* Correctly show file names of GZIP'ed assets. #1364
* Build file output is not parallel-ized! Use `middleman build --no-parallel` to disable.
# 4.0.0

2
middleman-cli/bin/middleman
View file

@ -4,7 +4,7 @@ require 'middleman-core/profiling'
if ARGV.include? '--profile'
Middleman::Profiling.profiler = Middleman::Profiling::RubyProfProfiler.new
end
Middleman::Profiling.start
# Middleman::Profiling.start
require "middleman-core/load_paths"
Middleman.setup_load_paths

4
middleman-cli/lib/middleman-cli/build.rb
View file

@ -14,6 +14,10 @@ module Middleman::Cli
type: :boolean,
default: true,
desc: 'Remove orphaned files from build (--no-clean to disable)'
class_option :parallel,
type: :boolean,
default: true,
desc: 'Output files in parallel (--no-parallel to disable)'
class_option :glob,
type: :string,
aliases: '-g',

4
middleman-core/lib/middleman-core/application.rb
View file

@ -184,7 +184,7 @@ module Middleman
end
}, 'Callbacks that can exclude paths from the sitemap'
define_setting :skip_build_clean, proc { |p| [/\.git/].any? { |r| r.match(p) } }, 'Whether some paths should not be removed during a clean build.'
define_setting :skip_build_clean, proc { |p| [/\.git/].any? { |r| p =~ r } }, 'Whether some paths should not be removed during a clean build.'
define_setting :watcher_disable, false, 'If the Listen watcher should not run'
define_setting :watcher_force_polling, false, 'If the Listen watcher should run in polling mode'
@ -314,7 +314,7 @@ module Middleman
# Clean up missing Tilt exts
def prune_tilt_templates!
::Tilt.mappings.each do |key, _|
::Tilt.mappings.each_key do |key|
begin
::Tilt[".#{key}"]
rescue LoadError, NameError

32
middleman-core/lib/middleman-core/builder.rb
View file

@ -1,6 +1,7 @@
require 'pathname'
require 'fileutils'
require 'tempfile'
require 'parallel'
require 'middleman-core/rack'
require 'middleman-core/callback_manager'
require 'middleman-core/contracts'
@ -36,6 +37,7 @@ module Middleman
@glob = opts.fetch(:glob)
@cleaning = opts.fetch(:clean)
@parallel = opts.fetch(:parallel, true)
rack_app = ::Middleman::Rack.new(@app).to_app
@rack = ::Rack::MockRequest.new(rack_app)
@ -63,16 +65,18 @@ module Middleman
prerender_css
end
::Middleman::Profiling.start
::Middleman::Util.instrument 'builder.output' do
output_files
end
::Middleman::Profiling.report('build')
::Middleman::Util.instrument 'builder.clean' do
clean! if @cleaning
end
::Middleman::Profiling.report('build')
::Middleman::Util.instrument 'builder.after' do
@app.execute_callbacks(:after_build, [self])
end
@ -87,9 +91,8 @@ module Middleman
logger.debug '== Prerendering CSS'
css_files = ::Middleman::Util.instrument 'builder.prerender.output' do
@app.sitemap.resources
.select { |resource| resource.ext == '.css' }
.each(&method(:output_resource))
resources = @app.sitemap.resources.select { |resource| resource.ext == '.css' }
output_resources(resources)
end
::Middleman::Util.instrument 'builder.prerender.check-files' do
@ -117,7 +120,20 @@ module Middleman
resources = resources.select { |resource| File.fnmatch(@glob, resource.destination_path) }
end
resources.each(&method(:output_resource))
output_resources(resources)
end
Contract ResourceList => ResourceList
def output_resources(resources)
cleaned_paths = if @parallel
::Parallel.map(resources, &method(:output_resource))
else
resources.map(&method(:output_resource))
end
cleaned_paths.each { |p| @to_clean.delete(p) } if @cleaning
resources
end
# Figure out the correct event mode.
@ -181,7 +197,7 @@ module Middleman
# Try to output a resource and capture errors.
# @param [Middleman::Sitemap::Resource] resource The resource.
# @return [void]
Contract IsA['Middleman::Sitemap::Resource'] => Any
Contract IsA['Middleman::Sitemap::Resource'] => Maybe[Pathname]
def output_resource(resource)
output_file = nil
@ -218,7 +234,7 @@ module Middleman
output_file
end
@to_clean.delete(Pathname(cleaned_name))
Pathname(cleaned_name)
end
# Get a list of all the paths in the destination folder and save them

4
middleman-core/lib/middleman-core/core_extensions/data.rb
View file

@ -203,11 +203,11 @@ module Middleman
def to_h
data = {}
store.each do |k, _|
store.each_key do |k|
data[k] = data_for_path(k)
end
callbacks.each do |k, _|
callbacks.each_key do |k|
data[k] = data_for_path(k)
end

12
middleman-core/lib/middleman-core/core_extensions/default_helpers.rb
View file

@ -113,7 +113,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
path_options = {}
path_options[:relative] = options.delete(:relative) if options.key?(:relative)
sources.flatten.inject(::ActiveSupport::SafeBuffer.new) do |all, source|
sources.flatten.reduce(::ActiveSupport::SafeBuffer.new) do |all, source|
all << tag(:link, {
href: asset_path(:css, source, path_options)
}.update(options))
@ -127,7 +127,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
path_options = {}
path_options[:relative] = options.delete(:relative) if options.key?(:relative)
sources.flatten.inject(::ActiveSupport::SafeBuffer.new) do |all, source|
sources.flatten.reduce(::ActiveSupport::SafeBuffer.new) do |all, source|
all << content_tag(:script, nil, {
src: asset_path(:js, source, path_options)
}.update(options))
@ -152,7 +152,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
# If the basename of the request as no extension, assume we are serving a
# directory and join index_file to the path.
path = File.join(asset_dir, current_resource.path)
path = path.sub(/#{Regexp.escape(File.extname(path))}$/, ".#{asset_ext}")
path = path[0..-(File.extname(path).length + 1)] + ".#{asset_ext}"
yield path if sitemap.find_resource_by_path(path)
end
@ -191,7 +191,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
# @param [Hash] options Data to pass through.
# @return [String]
def asset_path(kind, source, options={})
options_with_resource = options.merge(current_resource: current_resource)
options_with_resource = {}.merge!(options).merge!(current_resource: current_resource)
::Middleman::Util.asset_path(app, kind, source, options_with_resource)
end
@ -202,7 +202,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
# @param [Hash] options Additional options.
# @return [String] The fully qualified asset url
def asset_url(path, prefix='', options={})
options_with_resource = options.merge(current_resource: current_resource)
options_with_resource = {}.merge!(options).merge!(current_resource: current_resource)
::Middleman::Util.asset_url(app, path, prefix, options_with_resource)
end
@ -210,7 +210,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
# or a Resource, this will produce the nice URL configured for that
# path, respecting :relative_links, directory indexes, etc.
def url_for(path_or_resource, options={})
options_with_resource = options.merge(current_resource: current_resource)
options_with_resource = {}.merge!(options).merge!(current_resource: current_resource)
::Middleman::Util.url_for(app, path_or_resource, options_with_resource)
end

2
middleman-core/lib/middleman-core/core_extensions/i18n.rb
View file

@ -203,7 +203,7 @@ class Middleman::CoreExtensions::Internationalization < ::Middleman::Extension
if (options[:mount_at_root] == locale) || (options[:mount_at_root].nil? && locales[0] == locale)
'/'
else
replacement = options[:locale_map].fetch(locale, locale)
replacement = options[:locale_map][locale] || locale
options[:path].sub(':locale', replacement.to_s).sub(':lang', replacement.to_s) # Backward compat
end
end

4
middleman-core/lib/middleman-core/core_extensions/inline_url_rewriter.rb
View file

@ -108,7 +108,7 @@ module Middleman
ignore = rewriter.fetch(:ignore)
next if ignore.any? { |r| should_ignore?(r, full_asset_path) }
rewrite_ignore = Array(rewriter.fetch(:rewrite_ignore, []))
rewrite_ignore = Array(rewriter[:rewrite_ignore] || [])
next if rewrite_ignore.any? { |i| ::Middleman::Util.path_match(i, path) }
proc = rewriter.fetch(:proc)
@ -132,7 +132,7 @@ module Middleman
def should_ignore?(validator, value)
if validator.is_a? Regexp
# Treat as Regexp
!value.match(validator).nil?
!!(value =~ validator)
elsif validator.respond_to? :call
# Treat as proc
validator.call(value)

2
middleman-core/lib/middleman-core/extension.rb
View file

@ -432,7 +432,7 @@ module Middleman
{}
end
sum.merge(resource_definitions)
sum.merge!(resource_definitions)
end
resources + generator_defs.map do |path, g|

2
middleman-core/lib/middleman-core/extensions.rb
View file

@ -106,7 +106,7 @@ module Middleman
# A flattened list of all extensions which are automatically activated
# @return [Array<Symbol>] A list of extension names which are automatically activated.
def auto_activated
@auto_activate.values.map(&:to_a).flatten.map(&:name)
@auto_activate.values.map(&:to_a).flat_map(&:name)
end
# @api private

2
middleman-core/lib/middleman-core/extensions/automatic_image_sizes.rb
View file

@ -19,7 +19,7 @@ class Middleman::Extensions::AutomaticImageSizes < ::Middleman::Extension
real_path = path.dup
real_path = File.join(config[:images_dir], real_path) unless real_path.start_with?('/')
file = app.files.find(:source, real_path) || app.files.find(:source, real_path.gsub(/^\//, ''))
file = app.files.find(:source, real_path) || app.files.find(:source, real_path.sub(/^\//, ''))
if file && file[:full_path].exist?
begin

2
middleman-core/lib/middleman-core/extensions/minify_css.rb
View file

@ -23,7 +23,7 @@ class Middleman::Extensions::MinifyCss < ::Middleman::Extension
class SassCompressor
def self.compress(style, options={})
root_node = ::Sass::SCSS::CssParser.new(style, 'middleman-css-input', 1).parse
root_node.options = options.merge(style: :compressed)
root_node.options = {}.merge!(options).merge!(style: :compressed)
root_node.render.strip
end
end

2
middleman-core/lib/middleman-core/file_renderer.rb
View file

@ -51,7 +51,7 @@ module Middleman
# Merge per-extension options from config
extension = File.extname(path)
options = opts.merge(options_for_ext(extension))
options = {}.merge!(opts).merge!(options_for_ext(extension))
options[:outvar] ||= '@_out_buf'
options[:context] = context
options.delete(:layout)

2
middleman-core/lib/middleman-core/renderers/haml.rb
View file

@ -30,7 +30,7 @@ module Middleman
end
def evaluate(scope, locals, &block)
options = @options.merge(filename: eval_file, line: line, context: @context || scope)
options = {}.merge!(@options).merge!(filename: eval_file, line: line, context: @context || scope)
@engine = ::Haml::Engine.new(data, options)
output = @engine.render(scope, locals, &block)

2
middleman-core/lib/middleman-core/renderers/less.rb
View file

@ -26,7 +26,7 @@ module Middleman
if ::Less.const_defined? :Engine
@engine = ::Less::Engine.new(data)
else
parser = ::Less::Parser.new(options.merge(filename: eval_file, line: line, paths: ['.', File.dirname(eval_file)]))
parser = ::Less::Parser.new({}.merge!(options).merge!(filename: eval_file, line: line, paths: ['.', File.dirname(eval_file)]))
@engine = parser.parse(data)
end
end

4
middleman-core/lib/middleman-core/renderers/sass.rb
View file

@ -81,7 +81,7 @@ module Middleman
filename: eval_file,
line: line,
syntax: syntax,
custom: (options[:custom] || {}).merge(
custom: {}.merge!(options[:custom] || {}).merge!(
middleman_context: ctx.app,
current_resource: ctx.current_resource
)
@ -97,7 +97,7 @@ module Middleman
more_opts[:css_filename] = file.sub(/\.s[ac]ss$/, '')
end
options.merge(more_opts)
{}.merge!(options).merge!(more_opts)
end
end

11
middleman-core/lib/middleman-core/sources.rb
View file

@ -189,6 +189,7 @@ module Middleman
array_of_types = Array(types)
watchers
.lazy
.select { |d| array_of_types.include?(d.type) }
.map { |d| d.find(path, glob) }
.reject(&:nil?)
@ -202,9 +203,7 @@ module Middleman
# @return [Boolean]
Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String => Bool
def exists?(types, path)
watchers
.select { |d| Array(types).include?(d.type) }
.any? { |d| d.exists?(path) }
watchers.any? { |d| Array(types).include?(d.type) && d.exists?(path) }
end
# Check if a file for a given type exists.
@ -214,9 +213,7 @@ module Middleman
# @return [Boolean]
Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String => Maybe[HANDLER]
def watcher_for_path(types, path)
watchers
.select { |d| Array(types).include?(d.type) }
.find { |d| d.exists?(path) }
watchers.detect { |d| Array(types).include?(d.type) && d.exists?(path) }
end
# Manually check for new files
@ -317,7 +314,7 @@ module Middleman
def matches?(validator, file)
path = file[:relative_path]
if validator.is_a? Regexp
!!validator.match(path.to_s)
!!(path.to_s =~ validator)
else
!!validator.call(path, @app)
end

2
middleman-core/lib/middleman-core/sources/source_watcher.rb
View file

@ -322,7 +322,7 @@ module Middleman
if @only.empty?
!@ignored.call(file)
else
@only.any? { |reg| reg.match(file[:relative_path].to_s) }
@only.any? { |reg| file[:relative_path].to_s =~ reg }
end
end
end

9
middleman-core/lib/middleman-core/util.rb
View file

@ -79,7 +79,7 @@ module Middleman
path == matcher
end
when matcher.respond_to?(:match)
!matcher.match(path).nil?
!!(path =~ matcher)
when matcher.respond_to?(:call)
matcher.call(path)
else
@ -357,7 +357,7 @@ module Middleman
begin
uri = ::Addressable::URI.parse(asset_path)
if uri.relative? && uri.host.nil? && !asset_path.match(/^[^\/].*[a-z]+\.[a-z]+\/.*/) && (result = yield(asset_path))
if uri.relative? && uri.host.nil? && !(asset_path =~ /^[^\/].*[a-z]+\.[a-z]+\/.*/) && (result = yield(asset_path))
"#{opening_character}#{result}"
else
match
@ -456,10 +456,11 @@ module Middleman
Contract String => String
def step_through_extensions(path)
while ::Tilt[path]
yield File.extname(path) if block_given?
ext = File.extname(path)
yield ext if block_given?
# Strip templating extensions as long as Tilt knows them
path = path.sub(/#{::Regexp.escape(File.extname(path))}$/, '')
path = path[0..-(ext.length + 1)]
end
yield File.extname(path) if block_given?

1
middleman-core/middleman-core.gemspec
View file

@ -24,6 +24,7 @@ Gem::Specification.new do |s|
s.add_dependency('tilt', ['~> 1.4.1'])
s.add_dependency('erubis')
s.add_dependency('fast_blank')
s.add_dependency('parallel')
# Helpers
s.add_dependency('activesupport', ['~> 4.2'])