From 3b413355822ef7f093a6dc5f06c37dda61c7778c Mon Sep 17 00:00:00 2001 From: Thomas Reynolds Date: Mon, 28 Mar 2016 17:13:55 -0700 Subject: [PATCH] WIP manifest --- .gitignore | 1 + Gemfile | 6 +- middleman-cli/lib/middleman-cli/build.rb | 7 +- .../related-files-app/source/index.html.erb | 1 + .../lib/middleman-core/application.rb | 4 +- middleman-core/lib/middleman-core/builder.rb | 118 ++++++++++++++++-- .../lib/middleman-core/template_context.rb | 2 + .../lib/middleman-core/template_renderer.rb | 2 + 8 files changed, 128 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index aabe272b..87f495a6 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ docs .ruby-gemset .*.swp build +manifest.yaml doc .yardoc tmp diff --git a/Gemfile b/Gemfile index f14eb3b1..35627a0b 100644 --- a/Gemfile +++ b/Gemfile @@ -25,12 +25,12 @@ gem 'redcarpet', '>= 3.1', require: false gem 'rubydns', '~> 1.0.1', require: false # To test javascript -gem 'poltergeist', '~> 1.8', require: false -gem 'phantomjs', '~> 1.9.8.0', require: false +gem 'poltergeist', '~> 1.9', require: false +gem 'phantomjs', '~> 2.1.1', require: false # For less, note there is no compatible JS runtime for windows gem 'therubyrhino', '>= 2.0', platforms: :jruby -gem 'therubyracer', '>= 0.12', platforms: :ruby +gem 'therubyracer', '>= 0.12.1', platforms: :ruby # Code Quality gem 'rubocop', '~> 0.24', require: false diff --git a/middleman-cli/lib/middleman-cli/build.rb b/middleman-cli/lib/middleman-cli/build.rb index 7d5dd3af..54243f34 100644 --- a/middleman-cli/lib/middleman-cli/build.rb +++ b/middleman-cli/lib/middleman-cli/build.rb @@ -19,6 +19,10 @@ module Middleman::Cli type: :boolean, default: true, desc: 'Output files in parallel (--no-parallel to disable)' + class_option :manifest, + type: :boolean, + default: false, + desc: 'Use a manifest.yaml to optimize incremental builds' class_option :glob, type: :string, aliases: '-g', @@ -71,7 +75,8 @@ module Middleman::Cli builder = Middleman::Builder.new(@app, glob: options['glob'], clean: options['clean'], - parallel: options['parallel']) + parallel: options['parallel'], + manifest: options['manifest']) builder.thor = self builder.on_build_event(&method(:on_event)) end diff --git a/middleman-core/fixtures/related-files-app/source/index.html.erb b/middleman-core/fixtures/related-files-app/source/index.html.erb index e69de29b..6d37644c 100644 --- a/middleman-core/fixtures/related-files-app/source/index.html.erb +++ b/middleman-core/fixtures/related-files-app/source/index.html.erb @@ -0,0 +1 @@ +<%= partial "/partials/test" %> diff --git a/middleman-core/lib/middleman-core/application.rb b/middleman-core/lib/middleman-core/application.rb index 379d1e81..298732eb 100644 --- a/middleman-core/lib/middleman-core/application.rb +++ b/middleman-core/lib/middleman-core/application.rb @@ -236,7 +236,9 @@ module Middleman :before_render, :after_render, :before_server, - :reload + :reload, + :render_layout, + :render_partial ]) @middleware = Set.new diff --git a/middleman-core/lib/middleman-core/builder.rb b/middleman-core/lib/middleman-core/builder.rb index 1796d4d7..c58ab655 100644 --- a/middleman-core/lib/middleman-core/builder.rb +++ b/middleman-core/lib/middleman-core/builder.rb @@ -2,6 +2,8 @@ require 'pathname' require 'fileutils' require 'tempfile' require 'parallel' +require 'set' +require 'digest/sha1' require 'middleman-core/rack' require 'middleman-core/callback_manager' require 'middleman-core/contracts' @@ -11,6 +13,8 @@ module Middleman extend Forwardable include Contracts + OutputResult = Struct.new(:path, :layouts, :partials) + # Make app & events available to `after_build` callbacks. attr_reader :app, :events @@ -23,6 +27,8 @@ module Middleman # Sort order, images, fonts, js/css and finally everything else. SORT_ORDER = %w(.png .jpeg .jpg .gif .bmp .svg .svgz .webp .ico .woff .woff2 .otf .ttf .eot .js .css).freeze + MANIFEST_FILE = 'manifest.yaml' + # Create a new Builder instance. # @param [Middleman::Application] app The app to build. # @param [Hash] opts The builder options @@ -38,6 +44,11 @@ module Middleman @glob = opts.fetch(:glob) @cleaning = opts.fetch(:clean) @parallel = opts.fetch(:parallel, true) + @manifest = opts.fetch(:manifest, false) + @cleaning = @glob = false if @manifest + + @layout_files = Set.new + @partial_files = Set.new rack_app = ::Middleman::Rack.new(@app).to_app @rack = ::Rack::MockRequest.new(rack_app) @@ -62,13 +73,20 @@ module Middleman end ::Middleman::Util.instrument 'builder.prerender' do - prerender_css + prerender_css unless @manifest end ::Middleman::Profiling.start ::Middleman::Util.instrument 'builder.output' do - output_files + if @manifest && m = load_manifest! + incremental_resources = calculate_incremental(m) + + logger.debug '== Building files' + output_resources(incremental_resources) + else + output_files + end end ::Middleman::Profiling.report('build') @@ -81,6 +99,8 @@ module Middleman @app.execute_callbacks(:after_build, [self]) end + build_manifest! if @manifest && !@has_error + !@has_error end @@ -137,17 +157,20 @@ module Middleman resources.map(&method(:output_resource)) end + @partial_files = results.reduce(Set.new) { |sum, r| r ? sum + r[:partials] : sum } + @layout_files = results.reduce(Set.new) { |sum, r| r ? sum + r[:layouts] : sum } + @has_error = true if results.any? { |r| r == false } if @cleaning && !@has_error results.each do |p| - next unless p.exist? + next unless p[:path].exist? # handle UTF-8-MAC filename on MacOS cleaned_name = if RUBY_PLATFORM =~ /darwin/ - p.to_s.encode('UTF-8', 'UTF-8-MAC') + p[:path].to_s.encode('UTF-8', 'UTF-8-MAC') else - p + p[:path] end @to_clean.delete(Pathname(cleaned_name)) @@ -166,7 +189,7 @@ module Middleman if !output_file.exist? :created else - FileUtils.compare_file(source.to_s, output_file.to_s) ? :identical : :updated + ::FileUtils.compare_file(source.to_s, output_file.to_s) ? :identical : :updated end end @@ -218,11 +241,22 @@ module Middleman # Try to output a resource and capture errors. # @param [Middleman::Sitemap::Resource] resource The resource. # @return [void] - Contract IsA['Middleman::Sitemap::Resource'] => Or[Pathname, Bool] + Contract IsA['Middleman::Sitemap::Resource'] => Or[OutputResult, Bool] def output_resource(resource) ::Middleman::Util.instrument 'builder.output.resource', path: File.basename(resource.destination_path) do output_file = @build_dir + resource.destination_path.gsub('%20', ' ') + layouts = Set.new + partials = Set.new + + @app.render_layout do |f| + layouts << f[:full_path] + end + + @app.render_partial do |f| + partials << f[:full_path] + end + begin if resource.binary? export_file!(output_file, resource.file_descriptor[:full_path]) @@ -242,7 +276,7 @@ module Middleman return false end - output_file + OutputResult.new(output_file, layouts, partials) end end @@ -299,5 +333,73 @@ module Middleman execute_callbacks(:on_build_event, [event_type, target, extra]) end + + def load_manifest! + return nil unless File.exist?(MANIFEST_FILE) + + m = ::YAML.load(File.read(MANIFEST_FILE)) + + all_files = Set.new(@app.files.files.map { |f| f[:full_path].relative_path_from(@app.root_path).to_s }) + ruby_files = Set.new(Dir[File.join(app.root, "**/*.rb")].map { |f| Pathname(f).relative_path_from(@app.root_path).to_s }) << "Gemfile.lock" + + locales_path = app.extensions[:i18n] && app.extensions[:i18n].options[:data] + + partial_and_layout_files = all_files.select do |f| + f.start_with?(app.config[:layouts_dir] + "/") || f.split("/").any? { |d| d.start_with?("_") } || (locales_path && f.start_with?(locales_path + "/")) + end + + @global_paths = Set.new(partial_and_layout_files) + ruby_files + + (all_files + ruby_files).select do |f| + if m[f] + dig = ::Digest::SHA1.file(f).to_s + dig != m[f] + else + true + end + end + rescue StandardError, ::Psych::SyntaxError => error + logger.error "Manifest file (#{MANIFEST_FILE}) was malformed." + end + + def calculate_incremental(manifest) + logger.debug '== Calculating incremental build files' + + if manifest.empty? + logger.debug "== No files changed" + return [] + end + + resources = @app.sitemap.resources.sort_by { |resource| SORT_ORDER.index(resource.ext) || 100 } + + if changed = manifest.select { |p| @global_paths.include?(p) } + if changed.length > 0 + logger.debug "== Global file changed: #{changed}" + return resources + end + end + + resources.select do |r| + if r.file_descriptor + path = r.file_descriptor[:full_path].relative_path_from(@app.root_path).to_s + manifest.include?(path) + else + false + end + end + end + + def build_manifest! + all_files = Set.new(@app.files.files.map { |f| f[:full_path].to_s }) + ruby_files = Set.new(Dir[File.join(app.root, "**/*.rb")]) << File.expand_path("Gemfile.lock", @app.root) + + manifest = (all_files + ruby_files).each_with_object({}) do |source_file, sum| + path = Pathname(source_file).relative_path_from(@app.root_path).to_s + sum[path] = ::Digest::SHA1.file(source_file).to_s + end + + ::File.write(MANIFEST_FILE, manifest.to_yaml) + trigger(:created, MANIFEST_FILE) + end end end diff --git a/middleman-core/lib/middleman-core/template_context.rb b/middleman-core/lib/middleman-core/template_context.rb index 99ec6ee5..7fcd34da 100644 --- a/middleman-core/lib/middleman-core/template_context.rb +++ b/middleman-core/lib/middleman-core/template_context.rb @@ -149,6 +149,8 @@ module Middleman break if partial_file end + @app.execute_callbacks(:render_partial, [partial_file]) if partial_file + partial_file || nil end diff --git a/middleman-core/lib/middleman-core/template_renderer.rb b/middleman-core/lib/middleman-core/template_renderer.rb index 1930d434..495bcd2e 100644 --- a/middleman-core/lib/middleman-core/template_renderer.rb +++ b/middleman-core/lib/middleman-core/template_renderer.rb @@ -140,6 +140,8 @@ module Middleman # If we need a layout and have a layout, use it layout_file = fetch_layout(engine, options) if layout_file + @app.execute_callbacks(:render_layout, [layout_file]) + content = ::Middleman::Util.instrument 'builder.output.resource.render-layout', path: File.basename(layout_file[:relative_path].to_s) do if layout_file = fetch_layout(engine, options) layout_renderer = ::Middleman::FileRenderer.new(@app, layout_file[:relative_path].to_s)