WIP manifest

.gitignore

@@ -16,6 +16,7 @@ docs
 .ruby-gemset
 .*.swp
 build
+manifest.yaml
 doc
 .yardoc
 tmp

Gemfile

@@ -25,12 +25,12 @@ gem 'redcarpet', '>= 3.1', require: false
 gem 'rubydns', '~> 1.0.1', require: false
 
 # To test javascript
-gem 'poltergeist', '~> 1.8', require: false
+gem 'poltergeist', '~> 1.9', require: false
-gem 'phantomjs', '~> 1.9.8.0', require: false
+gem 'phantomjs', '~> 2.1.1', require: false
 
 # For less, note there is no compatible JS runtime for windows
 gem 'therubyrhino', '>= 2.0', platforms: :jruby
-gem 'therubyracer', '>= 0.12', platforms: :ruby
+gem 'therubyracer', '>= 0.12.1', platforms: :ruby
 
 # Code Quality
 gem 'rubocop', '~> 0.24', require: false

middleman-cli/lib/middleman-cli/build.rb

@@ -19,6 +19,10 @@ module Middleman::Cli
                  type: :boolean,
                  default: true,
                  desc: 'Output files in parallel (--no-parallel to disable)'
+    class_option :manifest,
+                 type: :boolean,
+                 default: false,
+                 desc: 'Use a manifest.yaml to optimize incremental builds'
     class_option :glob,
                  type: :string,
                  aliases: '-g',
@@ -71,7 +75,8 @@ module Middleman::Cli
         builder = Middleman::Builder.new(@app,
                                          glob: options['glob'],
                                          clean: options['clean'],
-                                         parallel: options['parallel'])
+                                         parallel: options['parallel'],
+                                         manifest: options['manifest'])
         builder.thor = self
         builder.on_build_event(&method(:on_event))
       end

middleman-core/fixtures/related-files-app/source/index.html.erb

@@ -0,0 +1 @@
+<%= partial "/partials/test" %>

middleman-core/lib/middleman-core/application.rb

@@ -236,7 +236,9 @@ module Middleman
                                       :before_render,
                                       :after_render,
                                       :before_server,
-                                      :reload
+                                      :reload,
+                                      :render_layout,
+                                      :render_partial
                                     ])
 
         @middleware = Set.new

middleman-core/lib/middleman-core/builder.rb

@@ -2,6 +2,8 @@ require 'pathname'
 require 'fileutils'
 require 'tempfile'
 require 'parallel'
+require 'set'
+require 'digest/sha1'
 require 'middleman-core/rack'
 require 'middleman-core/callback_manager'
 require 'middleman-core/contracts'
@@ -11,6 +13,8 @@ module Middleman
     extend Forwardable
     include Contracts
 
+    OutputResult = Struct.new(:path, :layouts, :partials)
+
     # Make app & events available to `after_build` callbacks.
     attr_reader :app, :events
 
@@ -23,6 +27,8 @@ module Middleman
     # Sort order, images, fonts, js/css and finally everything else.
     SORT_ORDER = %w(.png .jpeg .jpg .gif .bmp .svg .svgz .webp .ico .woff .woff2 .otf .ttf .eot .js .css).freeze
 
+    MANIFEST_FILE = 'manifest.yaml'
+
     # Create a new Builder instance.
     # @param [Middleman::Application] app The app to build.
     # @param [Hash] opts The builder options
@@ -38,6 +44,11 @@ module Middleman
       @glob = opts.fetch(:glob)
       @cleaning = opts.fetch(:clean)
       @parallel = opts.fetch(:parallel, true)
+      @manifest = opts.fetch(:manifest, false)
+      @cleaning = @glob = false if @manifest
+
+      @layout_files = Set.new
+      @partial_files = Set.new
 
       rack_app = ::Middleman::Rack.new(@app).to_app
       @rack = ::Rack::MockRequest.new(rack_app)
@@ -62,13 +73,20 @@ module Middleman
       end
 
       ::Middleman::Util.instrument 'builder.prerender' do
-        prerender_css
+        prerender_css unless @manifest
       end
 
       ::Middleman::Profiling.start
 
       ::Middleman::Util.instrument 'builder.output' do
-        output_files
+        if @manifest && m = load_manifest!
+          incremental_resources = calculate_incremental(m)
+
+          logger.debug '== Building files'
+          output_resources(incremental_resources)
+        else
+          output_files
+        end
       end
 
       ::Middleman::Profiling.report('build')
@@ -81,6 +99,8 @@ module Middleman
         @app.execute_callbacks(:after_build, [self])
       end
 
+      build_manifest! if @manifest && !@has_error
+
       !@has_error
     end
 
@@ -137,17 +157,20 @@ module Middleman
         resources.map(&method(:output_resource))
       end
 
+      @partial_files = results.reduce(Set.new) { |sum, r| r ? sum + r[:partials] : sum }
+      @layout_files = results.reduce(Set.new) { |sum, r| r ? sum + r[:layouts] : sum }
+
       @has_error = true if results.any? { |r| r == false }
 
       if @cleaning && !@has_error
         results.each do |p|
-          next unless p.exist?
+          next unless p[:path].exist?
 
           # handle UTF-8-MAC filename on MacOS
           cleaned_name = if RUBY_PLATFORM =~ /darwin/
-            p.to_s.encode('UTF-8', 'UTF-8-MAC')
+            p[:path].to_s.encode('UTF-8', 'UTF-8-MAC')
           else
-            p
+            p[:path]
           end
 
           @to_clean.delete(Pathname(cleaned_name))
@@ -166,7 +189,7 @@ module Middleman
       if !output_file.exist?
         :created
       else
-        FileUtils.compare_file(source.to_s, output_file.to_s) ? :identical : :updated
+        ::FileUtils.compare_file(source.to_s, output_file.to_s) ? :identical : :updated
       end
     end
 
@@ -218,11 +241,22 @@ module Middleman
     # Try to output a resource and capture errors.
     # @param [Middleman::Sitemap::Resource] resource The resource.
     # @return [void]
-    Contract IsA['Middleman::Sitemap::Resource'] => Or[Pathname, Bool]
+    Contract IsA['Middleman::Sitemap::Resource'] => Or[OutputResult, Bool]
     def output_resource(resource)
       ::Middleman::Util.instrument 'builder.output.resource', path: File.basename(resource.destination_path) do
         output_file = @build_dir + resource.destination_path.gsub('%20', ' ')
 
+        layouts = Set.new
+        partials = Set.new
+
+        @app.render_layout do |f|
+          layouts << f[:full_path]
+        end
+
+        @app.render_partial do |f|
+          partials << f[:full_path]
+        end
+
         begin
           if resource.binary?
             export_file!(output_file, resource.file_descriptor[:full_path])
@@ -242,7 +276,7 @@ module Middleman
           return false
         end
 
-        output_file
+        OutputResult.new(output_file, layouts, partials)
       end
     end
 
@@ -299,5 +333,73 @@ module Middleman
 
       execute_callbacks(:on_build_event, [event_type, target, extra])
     end
+
+    def load_manifest!
+      return nil unless File.exist?(MANIFEST_FILE)
+
+      m = ::YAML.load(File.read(MANIFEST_FILE))
+
+      all_files = Set.new(@app.files.files.map { |f| f[:full_path].relative_path_from(@app.root_path).to_s })
+      ruby_files = Set.new(Dir[File.join(app.root, "**/*.rb")].map { |f| Pathname(f).relative_path_from(@app.root_path).to_s }) << "Gemfile.lock"
+
+      locales_path = app.extensions[:i18n] && app.extensions[:i18n].options[:data]
+
+      partial_and_layout_files = all_files.select do |f|
+        f.start_with?(app.config[:layouts_dir] + "/") || f.split("/").any? { |d| d.start_with?("_") } || (locales_path && f.start_with?(locales_path + "/"))
+      end
+
+      @global_paths = Set.new(partial_and_layout_files) + ruby_files
+
+      (all_files + ruby_files).select do |f|
+        if m[f]
+          dig = ::Digest::SHA1.file(f).to_s
+          dig != m[f]
+        else
+          true
+        end
+      end
+    rescue StandardError, ::Psych::SyntaxError => error
+      logger.error "Manifest file (#{MANIFEST_FILE}) was malformed."
+    end
+
+    def calculate_incremental(manifest)
+      logger.debug '== Calculating incremental build files'
+
+      if manifest.empty?
+        logger.debug "== No files changed"
+        return []
+      end
+
+      resources = @app.sitemap.resources.sort_by { |resource| SORT_ORDER.index(resource.ext) || 100 }
+
+      if changed = manifest.select { |p| @global_paths.include?(p) }
+        if changed.length > 0
+          logger.debug "== Global file changed: #{changed}"
+          return resources
+        end
+      end
+
+      resources.select do |r|
+        if r.file_descriptor
+          path = r.file_descriptor[:full_path].relative_path_from(@app.root_path).to_s
+          manifest.include?(path)
+        else
+          false
+        end
+      end
+    end
+
+    def build_manifest!
+      all_files = Set.new(@app.files.files.map { |f| f[:full_path].to_s })
+      ruby_files = Set.new(Dir[File.join(app.root, "**/*.rb")]) << File.expand_path("Gemfile.lock", @app.root)
+
+      manifest = (all_files + ruby_files).each_with_object({}) do |source_file, sum|
+        path = Pathname(source_file).relative_path_from(@app.root_path).to_s
+        sum[path] = ::Digest::SHA1.file(source_file).to_s
+      end
+
+      ::File.write(MANIFEST_FILE, manifest.to_yaml)
+      trigger(:created, MANIFEST_FILE)
+    end
   end
 end

middleman-core/lib/middleman-core/template_context.rb

@@ -149,6 +149,8 @@ module Middleman
         break if partial_file
       end
 
+      @app.execute_callbacks(:render_partial, [partial_file]) if partial_file
+
       partial_file || nil
     end
 

middleman-core/lib/middleman-core/template_renderer.rb

@@ -140,6 +140,8 @@ module Middleman
       # If we need a layout and have a layout, use it
       layout_file = fetch_layout(engine, options)
       if layout_file
+        @app.execute_callbacks(:render_layout, [layout_file])
+
         content = ::Middleman::Util.instrument 'builder.output.resource.render-layout', path: File.basename(layout_file[:relative_path].to_s) do
           if layout_file = fetch_layout(engine, options)
             layout_renderer = ::Middleman::FileRenderer.new(@app, layout_file[:relative_path].to_s)