Perf work and Parallel builds

CHANGELOG.md

@@ -19,6 +19,7 @@ master
 * Fix asset_host in combination with Google Analytics snippet. #1751
 * Show an error message when git CLI is not available. #1765
 * Correctly show file names of GZIP'ed assets. #1364
+* Build file output is not parallel-ized! Use `middleman build --no-parallel` to disable.
 
 # 4.0.0
 

middleman-cli/bin/middleman

@@ -4,7 +4,7 @@ require 'middleman-core/profiling'
 if ARGV.include? '--profile'
   Middleman::Profiling.profiler = Middleman::Profiling::RubyProfProfiler.new
 end
-Middleman::Profiling.start
+# Middleman::Profiling.start
 
 require "middleman-core/load_paths"
 Middleman.setup_load_paths

middleman-cli/lib/middleman-cli/build.rb

@@ -14,6 +14,10 @@ module Middleman::Cli
                  type: :boolean,
                  default: true,
                  desc: 'Remove orphaned files from build (--no-clean to disable)'
+    class_option :parallel,
+                 type: :boolean,
+                 default: true,
+                 desc: 'Output files in parallel (--no-parallel to disable)'
     class_option :glob,
                  type: :string,
                  aliases: '-g',

middleman-core/lib/middleman-core/application.rb

@@ -184,7 +184,7 @@ module Middleman
       end
     }, 'Callbacks that can exclude paths from the sitemap'
 
-    define_setting :skip_build_clean, proc { |p| [/\.git/].any? { |r| r.match(p) } }, 'Whether some paths should not be removed during a clean build.'
+    define_setting :skip_build_clean, proc { |p| [/\.git/].any? { |r| p =~ r } }, 'Whether some paths should not be removed during a clean build.'
 
     define_setting :watcher_disable, false, 'If the Listen watcher should not run'
     define_setting :watcher_force_polling, false, 'If the Listen watcher should run in polling mode'
@@ -314,7 +314,7 @@ module Middleman
 
     # Clean up missing Tilt exts
     def prune_tilt_templates!
-      ::Tilt.mappings.each do |key, _|
+      ::Tilt.mappings.each_key do |key|
         begin
           ::Tilt[".#{key}"]
         rescue LoadError, NameError

middleman-core/lib/middleman-core/builder.rb

@@ -1,6 +1,7 @@
 require 'pathname'
 require 'fileutils'
 require 'tempfile'
+require 'parallel'
 require 'middleman-core/rack'
 require 'middleman-core/callback_manager'
 require 'middleman-core/contracts'
@@ -36,6 +37,7 @@ module Middleman
 
       @glob = opts.fetch(:glob)
       @cleaning = opts.fetch(:clean)
+      @parallel = opts.fetch(:parallel, true)
 
       rack_app = ::Middleman::Rack.new(@app).to_app
       @rack = ::Rack::MockRequest.new(rack_app)
@@ -63,16 +65,18 @@ module Middleman
         prerender_css
       end
 
+      ::Middleman::Profiling.start
+
       ::Middleman::Util.instrument 'builder.output' do
         output_files
       end
 
+      ::Middleman::Profiling.report('build')
+
       ::Middleman::Util.instrument 'builder.clean' do
         clean! if @cleaning
       end
 
-      ::Middleman::Profiling.report('build')
-
       ::Middleman::Util.instrument 'builder.after' do
         @app.execute_callbacks(:after_build, [self])
       end
@@ -87,9 +91,8 @@ module Middleman
       logger.debug '== Prerendering CSS'
 
       css_files = ::Middleman::Util.instrument 'builder.prerender.output' do
-        @app.sitemap.resources
+        resources = @app.sitemap.resources.select { |resource| resource.ext == '.css' }
-                        .select { |resource| resource.ext == '.css' }
+        output_resources(resources)
-                        .each(&method(:output_resource))
       end
 
       ::Middleman::Util.instrument 'builder.prerender.check-files' do
@@ -117,7 +120,20 @@ module Middleman
         resources = resources.select { |resource| File.fnmatch(@glob, resource.destination_path) }
       end
 
-      resources.each(&method(:output_resource))
+      output_resources(resources)
+    end
+
+    Contract ResourceList => ResourceList
+    def output_resources(resources)
+      cleaned_paths = if @parallel
+        ::Parallel.map(resources, &method(:output_resource))
+      else
+        resources.map(&method(:output_resource))
+      end
+
+      cleaned_paths.each { |p| @to_clean.delete(p) } if @cleaning
+
+      resources
     end
 
     # Figure out the correct event mode.
@@ -181,7 +197,7 @@ module Middleman
     # Try to output a resource and capture errors.
     # @param [Middleman::Sitemap::Resource] resource The resource.
     # @return [void]
-    Contract IsA['Middleman::Sitemap::Resource'] => Any
+    Contract IsA['Middleman::Sitemap::Resource'] => Maybe[Pathname]
     def output_resource(resource)
       output_file = nil
 
@@ -218,7 +234,7 @@ module Middleman
         output_file
       end
 
-      @to_clean.delete(Pathname(cleaned_name))
+      Pathname(cleaned_name)
     end
 
     # Get a list of all the paths in the destination folder and save them

middleman-core/lib/middleman-core/core_extensions/data.rb

@@ -203,11 +203,11 @@ module Middleman
         def to_h
           data = {}
 
-          store.each do |k, _|
+          store.each_key do |k|
             data[k] = data_for_path(k)
           end
 
-          callbacks.each do |k, _|
+          callbacks.each_key do |k|
             data[k] = data_for_path(k)
           end
 

middleman-core/lib/middleman-core/core_extensions/default_helpers.rb

@@ -113,7 +113,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
       path_options = {}
       path_options[:relative] = options.delete(:relative) if options.key?(:relative)
 
-      sources.flatten.inject(::ActiveSupport::SafeBuffer.new) do |all, source|
+      sources.flatten.reduce(::ActiveSupport::SafeBuffer.new) do |all, source|
         all << tag(:link, {
           href: asset_path(:css, source, path_options)
         }.update(options))
@@ -127,7 +127,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
       path_options = {}
       path_options[:relative] = options.delete(:relative) if options.key?(:relative)
 
-      sources.flatten.inject(::ActiveSupport::SafeBuffer.new) do |all, source|
+      sources.flatten.reduce(::ActiveSupport::SafeBuffer.new) do |all, source|
         all << content_tag(:script, nil, {
           src: asset_path(:js, source, path_options)
         }.update(options))
@@ -152,7 +152,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
       # If the basename of the request as no extension, assume we are serving a
       # directory and join index_file to the path.
       path = File.join(asset_dir, current_resource.path)
-      path = path.sub(/#{Regexp.escape(File.extname(path))}$/, ".#{asset_ext}")
+      path = path[0..-(File.extname(path).length + 1)] + ".#{asset_ext}"
 
       yield path if sitemap.find_resource_by_path(path)
     end
@@ -191,7 +191,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
     # @param [Hash] options Data to pass through.
     # @return [String]
     def asset_path(kind, source, options={})
-      options_with_resource = options.merge(current_resource: current_resource)
+      options_with_resource = {}.merge!(options).merge!(current_resource: current_resource)
       ::Middleman::Util.asset_path(app, kind, source, options_with_resource)
     end
 
@@ -202,7 +202,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
     # @param [Hash] options Additional options.
     # @return [String] The fully qualified asset url
     def asset_url(path, prefix='', options={})
-      options_with_resource = options.merge(current_resource: current_resource)
+      options_with_resource = {}.merge!(options).merge!(current_resource: current_resource)
       ::Middleman::Util.asset_url(app, path, prefix, options_with_resource)
     end
 
@@ -210,7 +210,7 @@ class Middleman::CoreExtensions::DefaultHelpers < ::Middleman::Extension
     # or a Resource, this will produce the nice URL configured for that
     # path, respecting :relative_links, directory indexes, etc.
     def url_for(path_or_resource, options={})
-      options_with_resource = options.merge(current_resource: current_resource)
+      options_with_resource = {}.merge!(options).merge!(current_resource: current_resource)
       ::Middleman::Util.url_for(app, path_or_resource, options_with_resource)
     end
 

middleman-core/lib/middleman-core/core_extensions/i18n.rb

@@ -203,7 +203,7 @@ class Middleman::CoreExtensions::Internationalization < ::Middleman::Extension
     if (options[:mount_at_root] == locale) || (options[:mount_at_root].nil? && locales[0] == locale)
       '/'
     else
-      replacement = options[:locale_map].fetch(locale, locale)
+      replacement = options[:locale_map][locale] || locale
       options[:path].sub(':locale', replacement.to_s).sub(':lang', replacement.to_s) # Backward compat
     end
   end

middleman-core/lib/middleman-core/core_extensions/inline_url_rewriter.rb

@@ -108,7 +108,7 @@ module Middleman
                 ignore = rewriter.fetch(:ignore)
                 next if ignore.any? { |r| should_ignore?(r, full_asset_path) }
 
-                rewrite_ignore = Array(rewriter.fetch(:rewrite_ignore, []))
+                rewrite_ignore = Array(rewriter[:rewrite_ignore] || [])
                 next if rewrite_ignore.any? { |i| ::Middleman::Util.path_match(i, path) }
 
                 proc = rewriter.fetch(:proc)
@@ -132,7 +132,7 @@ module Middleman
         def should_ignore?(validator, value)
           if validator.is_a? Regexp
             # Treat as Regexp
-            !value.match(validator).nil?
+            !!(value =~ validator)
           elsif validator.respond_to? :call
             # Treat as proc
             validator.call(value)

middleman-core/lib/middleman-core/extension.rb

@@ -432,7 +432,7 @@ module Middleman
           {}
         end
 
-        sum.merge(resource_definitions)
+        sum.merge!(resource_definitions)
       end
 
       resources + generator_defs.map do |path, g|

middleman-core/lib/middleman-core/extensions.rb

@@ -106,7 +106,7 @@ module Middleman
       # A flattened list of all extensions which are automatically activated
       # @return [Array<Symbol>] A list of extension names which are automatically activated.
       def auto_activated
-        @auto_activate.values.map(&:to_a).flatten.map(&:name)
+        @auto_activate.values.map(&:to_a).flat_map(&:name)
       end
 
       # @api private

middleman-core/lib/middleman-core/extensions/automatic_image_sizes.rb

@@ -19,7 +19,7 @@ class Middleman::Extensions::AutomaticImageSizes < ::Middleman::Extension
         real_path = path.dup
         real_path = File.join(config[:images_dir], real_path) unless real_path.start_with?('/')
 
-        file = app.files.find(:source, real_path) || app.files.find(:source, real_path.gsub(/^\//, ''))
+        file = app.files.find(:source, real_path) || app.files.find(:source, real_path.sub(/^\//, ''))
 
         if file && file[:full_path].exist?
           begin

middleman-core/lib/middleman-core/extensions/minify_css.rb

@@ -23,7 +23,7 @@ class Middleman::Extensions::MinifyCss < ::Middleman::Extension
   class SassCompressor
     def self.compress(style, options={})
       root_node = ::Sass::SCSS::CssParser.new(style, 'middleman-css-input', 1).parse
-      root_node.options = options.merge(style: :compressed)
+      root_node.options = {}.merge!(options).merge!(style: :compressed)
       root_node.render.strip
     end
   end

middleman-core/lib/middleman-core/file_renderer.rb

@@ -51,7 +51,7 @@ module Middleman
 
       # Merge per-extension options from config
       extension = File.extname(path)
-      options = opts.merge(options_for_ext(extension))
+      options = {}.merge!(opts).merge!(options_for_ext(extension))
       options[:outvar] ||= '@_out_buf'
       options[:context] = context
       options.delete(:layout)

middleman-core/lib/middleman-core/renderers/haml.rb

@@ -30,7 +30,7 @@ module Middleman
       end
 
       def evaluate(scope, locals, &block)
-        options = @options.merge(filename: eval_file, line: line, context: @context || scope)
+        options = {}.merge!(@options).merge!(filename: eval_file, line: line, context: @context || scope)
         @engine = ::Haml::Engine.new(data, options)
         output = @engine.render(scope, locals, &block)
 

middleman-core/lib/middleman-core/renderers/less.rb

@@ -26,7 +26,7 @@ module Middleman
           if ::Less.const_defined? :Engine
             @engine = ::Less::Engine.new(data)
           else
-            parser  = ::Less::Parser.new(options.merge(filename: eval_file, line: line, paths: ['.', File.dirname(eval_file)]))
+            parser = ::Less::Parser.new({}.merge!(options).merge!(filename: eval_file, line: line, paths: ['.', File.dirname(eval_file)]))
             @engine = parser.parse(data)
           end
         end

middleman-core/lib/middleman-core/renderers/sass.rb

@@ -81,7 +81,7 @@ module Middleman
             filename: eval_file,
             line: line,
             syntax: syntax,
-            custom: (options[:custom] || {}).merge(
+            custom: {}.merge!(options[:custom] || {}).merge!(
               middleman_context: ctx.app,
               current_resource: ctx.current_resource
             )
@@ -97,7 +97,7 @@ module Middleman
             more_opts[:css_filename] = file.sub(/\.s[ac]ss$/, '')
           end
 
-          options.merge(more_opts)
+          {}.merge!(options).merge!(more_opts)
         end
       end
 

middleman-core/lib/middleman-core/sources.rb

@@ -189,6 +189,7 @@ module Middleman
       array_of_types = Array(types)
 
       watchers
+        .lazy
         .select { |d| array_of_types.include?(d.type) }
         .map { |d| d.find(path, glob) }
         .reject(&:nil?)
@@ -202,9 +203,7 @@ module Middleman
     # @return [Boolean]
     Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String => Bool
     def exists?(types, path)
-      watchers
+      watchers.any? { |d| Array(types).include?(d.type) && d.exists?(path) }
-        .select { |d| Array(types).include?(d.type) }
-        .any? { |d| d.exists?(path) }
     end
 
     # Check if a file for a given type exists.
@@ -214,9 +213,7 @@ module Middleman
     # @return [Boolean]
     Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String => Maybe[HANDLER]
     def watcher_for_path(types, path)
-      watchers
+      watchers.detect { |d| Array(types).include?(d.type) && d.exists?(path) }
-        .select { |d| Array(types).include?(d.type) }
-        .find { |d| d.exists?(path) }
     end
 
     # Manually check for new files
@@ -317,7 +314,7 @@ module Middleman
     def matches?(validator, file)
       path = file[:relative_path]
       if validator.is_a? Regexp
-        !!validator.match(path.to_s)
+        !!(path.to_s =~ validator)
       else
         !!validator.call(path, @app)
       end

middleman-core/lib/middleman-core/sources/source_watcher.rb

@@ -322,7 +322,7 @@ module Middleman
       if @only.empty?
         !@ignored.call(file)
       else
-        @only.any? { |reg| reg.match(file[:relative_path].to_s) }
+        @only.any? { |reg| file[:relative_path].to_s =~ reg }
       end
     end
   end

middleman-core/lib/middleman-core/util.rb

@@ -79,7 +79,7 @@ module Middleman
           path == matcher
         end
       when matcher.respond_to?(:match)
-        !matcher.match(path).nil?
+        !!(path =~ matcher)
       when matcher.respond_to?(:call)
         matcher.call(path)
       else
@@ -357,7 +357,7 @@ module Middleman
         begin
           uri = ::Addressable::URI.parse(asset_path)
 
-          if uri.relative? && uri.host.nil? && !asset_path.match(/^[^\/].*[a-z]+\.[a-z]+\/.*/) && (result = yield(asset_path))
+          if uri.relative? && uri.host.nil? && !(asset_path =~ /^[^\/].*[a-z]+\.[a-z]+\/.*/) && (result = yield(asset_path))
             "#{opening_character}#{result}"
           else
             match
@@ -456,10 +456,11 @@ module Middleman
     Contract String => String
     def step_through_extensions(path)
       while ::Tilt[path]
-        yield File.extname(path) if block_given?
+        ext = File.extname(path)
+        yield ext if block_given?
 
         # Strip templating extensions as long as Tilt knows them
-        path = path.sub(/#{::Regexp.escape(File.extname(path))}$/, '')
+        path = path[0..-(ext.length + 1)]
       end
 
       yield File.extname(path) if block_given?

middleman-core/middleman-core.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |s|
   s.add_dependency('tilt', ['~> 1.4.1'])
   s.add_dependency('erubis')
   s.add_dependency('fast_blank')
+  s.add_dependency('parallel')
 
   # Helpers
   s.add_dependency('activesupport', ['~> 4.2'])