From ffa662a917aa7d605a5272d068623bf5e26e0799 Mon Sep 17 00:00:00 2001 From: Thomas Reynolds Date: Wed, 20 Jan 2016 11:50:25 -0800 Subject: [PATCH] Add system-wide file read cache --- middleman-core/features/import_files.feature | 2 - .../frontmatter-neighbor-app/config.rb | 2 +- .../config.rb | 2 +- .../middleman-core/core_extensions/data.rb | 4 +- .../core_extensions/front_matter.rb | 6 ++- .../lib/middleman-core/file_renderer.rb | 9 ++-- .../lib/middleman-core/preview_server.rb | 4 +- .../lib/middleman-core/renderers/liquid.rb | 2 +- .../lib/middleman-core/sitemap/resource.rb | 2 +- middleman-core/lib/middleman-core/sources.rb | 17 ++++++-- .../middleman-core/sources/source_watcher.rb | 30 +++++++++---- .../lib/middleman-core/template_context.rb | 2 +- middleman-core/lib/middleman-core/util.rb | 42 ++++++++----------- .../lib/middleman-core/util/data.rb | 7 ++-- 14 files changed, 76 insertions(+), 55 deletions(-) diff --git a/middleman-core/features/import_files.feature b/middleman-core/features/import_files.feature index b45f1256..faaf2039 100644 --- a/middleman-core/features/import_files.feature +++ b/middleman-core/features/import_files.feature @@ -13,5 +13,3 @@ Feature: Import files Then I should see 'jQuery' When I go to "/bower_components2/jquery/dist/jquery.js" Then I should see 'jQuery' - - diff --git a/middleman-core/fixtures/frontmatter-neighbor-app/config.rb b/middleman-core/fixtures/frontmatter-neighbor-app/config.rb index 6fa6e37c..019d65f7 100644 --- a/middleman-core/fixtures/frontmatter-neighbor-app/config.rb +++ b/middleman-core/fixtures/frontmatter-neighbor-app/config.rb @@ -15,7 +15,7 @@ class NeighborFrontmatter < ::Middleman::Extension next unless file - fmdata = ::Middleman::Util::Data.parse(file[:full_path], app.config[:frontmatter_delims], :yaml).first + fmdata = ::Middleman::Util::Data.parse(file, app.config[:frontmatter_delims], :yaml).first opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type) opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options) ignored = fmdata.delete(:ignored) diff --git a/middleman-core/fixtures/frontmatter-settings-neighbor-app/config.rb b/middleman-core/fixtures/frontmatter-settings-neighbor-app/config.rb index a7e0fc55..7adbe389 100644 --- a/middleman-core/fixtures/frontmatter-settings-neighbor-app/config.rb +++ b/middleman-core/fixtures/frontmatter-settings-neighbor-app/config.rb @@ -26,7 +26,7 @@ class NeighborFrontmatter < ::Middleman::Extension end def apply_neighbor_data(resource, file) - fmdata = ::Middleman::Util::Data.parse(file[:full_path], app.config[:frontmatter_delims], :yaml).first + fmdata = ::Middleman::Util::Data.parse(file, app.config[:frontmatter_delims], :yaml).first opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type) opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options) ignored = fmdata.delete(:ignored) diff --git a/middleman-core/lib/middleman-core/core_extensions/data.rb b/middleman-core/lib/middleman-core/core_extensions/data.rb index dca657ac..22fdd4a2 100644 --- a/middleman-core/lib/middleman-core/core_extensions/data.rb +++ b/middleman-core/lib/middleman-core/core_extensions/data.rb @@ -100,10 +100,10 @@ module Middleman basename = File.basename(data_path, extension) if %w(.yaml .yml).include?(extension) - data, postscript = ::Middleman::Util::Data.parse(file[:full_path], @app.config[:frontmatter_delims], :yaml) + data, postscript = ::Middleman::Util::Data.parse(file, @app.config[:frontmatter_delims], :yaml) data[:postscript] = postscript if !postscript.nil? && data.is_a?(Hash) elsif extension == '.json' - data, _postscript = ::Middleman::Util::Data.parse(file[:full_path], @app.config[:frontmatter_delims], :json) + data, _postscript = ::Middleman::Util::Data.parse(file, @app.config[:frontmatter_delims], :json) else return end diff --git a/middleman-core/lib/middleman-core/core_extensions/front_matter.rb b/middleman-core/lib/middleman-core/core_extensions/front_matter.rb index ea3f38af..8809004f 100644 --- a/middleman-core/lib/middleman-core/core_extensions/front_matter.rb +++ b/middleman-core/lib/middleman-core/core_extensions/front_matter.rb @@ -72,8 +72,10 @@ module Middleman::CoreExtensions return [{}, nil] unless file - @cache[file[:full_path]] ||= ::Middleman::Util::Data.parse( - file[:full_path], + return @cache[file[:full_path]] if @cache.key?(file[:full_path]) + + @cache[file[:full_path]] = ::Middleman::Util::Data.parse( + file, app.config[:frontmatter_delims] ) end diff --git a/middleman-core/lib/middleman-core/file_renderer.rb b/middleman-core/lib/middleman-core/file_renderer.rb index 09c6b5d2..3f385ed6 100644 --- a/middleman-core/lib/middleman-core/file_renderer.rb +++ b/middleman-core/lib/middleman-core/file_renderer.rb @@ -97,11 +97,12 @@ module Middleman Contract String def template_data_for_file if @app.extensions[:front_matter] - @app.extensions[:front_matter].template_data_for_file(@path) || File.read(@path) - else - file = @app.files.find(:source, @path) - file.read if file + result = @app.extensions[:front_matter].template_data_for_file(@path) + return result unless result.nil? end + + file = @app.files.find(:source, @path) + file ? file.read : File.read(@path) end protected diff --git a/middleman-core/lib/middleman-core/preview_server.rb b/middleman-core/lib/middleman-core/preview_server.rb index ce127b5f..acdb367d 100644 --- a/middleman-core/lib/middleman-core/preview_server.rb +++ b/middleman-core/lib/middleman-core/preview_server.rb @@ -235,8 +235,8 @@ module Middleman if ssl_certificate || ssl_private_key raise 'You must provide both :ssl_certificate and :ssl_private_key' unless ssl_private_key && ssl_certificate - http_opts[:SSLCertificate] = OpenSSL::X509::Certificate.new File.read ssl_certificate - http_opts[:SSLPrivateKey] = OpenSSL::PKey::RSA.new File.read ssl_private_key + http_opts[:SSLCertificate] = OpenSSL::X509::Certificate.new ::File.read ssl_certificate + http_opts[:SSLPrivateKey] = OpenSSL::PKey::RSA.new ::File.read ssl_private_key else # use a generated self-signed cert http_opts[:SSLCertName] = [ diff --git a/middleman-core/lib/middleman-core/renderers/liquid.rb b/middleman-core/lib/middleman-core/renderers/liquid.rb index c50258a9..e58528ed 100644 --- a/middleman-core/lib/middleman-core/renderers/liquid.rb +++ b/middleman-core/lib/middleman-core/renderers/liquid.rb @@ -14,7 +14,7 @@ module Middleman def read_template_file(template_path, _) file = app.files.find(:source, "_#{template_path}.liquid") raise ::Liquid::FileSystemError, "No such template '#{template_path}'" unless file - File.read(file[:full_path]) + file.read end # @return Array diff --git a/middleman-core/lib/middleman-core/sitemap/resource.rb b/middleman-core/lib/middleman-core/sitemap/resource.rb index 380b7faf..adb36cfe 100644 --- a/middleman-core/lib/middleman-core/sitemap/resource.rb +++ b/middleman-core/lib/middleman-core/sitemap/resource.rb @@ -54,7 +54,7 @@ module Middleman source = Pathname(source) if source && source.is_a?(String) @file_descriptor = if source && source.is_a?(Pathname) - ::Middleman::SourceFile.new(source.relative_path_from(@app.source_dir), source, @app.source_dir, Set.new([:source])) + ::Middleman::SourceFile.new(source.relative_path_from(@app.source_dir), source, @app.source_dir, Set.new([:source]), 0) else source end diff --git a/middleman-core/lib/middleman-core/sources.rb b/middleman-core/lib/middleman-core/sources.rb index 4b01d578..267a8069 100644 --- a/middleman-core/lib/middleman-core/sources.rb +++ b/middleman-core/lib/middleman-core/sources.rb @@ -3,7 +3,12 @@ require 'middleman-core/contracts' module Middleman # The standard "record" that contains information about a file on disk. - SourceFile = Struct.new :relative_path, :full_path, :directory, :types + SourceFile = Struct.new(:relative_path, :full_path, :directory, :types, :version) do + def read + ::Middleman::Sources.file_cache[full_path] ||= {} + ::Middleman::Sources.file_cache[full_path][version] ||= ::File.read(full_path) + end + end # Sources handle multiple on-disk collections of files which make up # a Middleman project. They are separated by `type` which can then be @@ -36,6 +41,8 @@ module Middleman # Reference to the global logger. def_delegator :@app, :logger + cattr_accessor :file_cache + # Built-in types # :source, :data, :locales, :reload @@ -50,6 +57,8 @@ module Middleman @watchers = watchers @sorted_watchers = @watchers.dup.freeze + ::Middleman::Sources.file_cache = {} + @options = options # Set of procs wanting to be notified of changes @@ -177,9 +186,10 @@ module Middleman # @return [Middleman::SourceFile, nil] Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String, Maybe[Bool] => Maybe[SourceFile] def find(types, path, glob=false) + array_of_types = Array(types) + watchers - .lazy - .select { |d| Array(types).include?(d.type) } + .select { |d| array_of_types.include?(d.type) } .map { |d| d.find(path, glob) } .reject(&:nil?) .first @@ -193,7 +203,6 @@ module Middleman Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String => Bool def exists?(types, path) watchers - .lazy .select { |d| Array(types).include?(d.type) } .any? { |d| d.exists?(path) } end diff --git a/middleman-core/lib/middleman-core/sources/source_watcher.rb b/middleman-core/lib/middleman-core/sources/source_watcher.rb index 1af9370c..84c1506b 100644 --- a/middleman-core/lib/middleman-core/sources/source_watcher.rb +++ b/middleman-core/lib/middleman-core/sources/source_watcher.rb @@ -228,11 +228,11 @@ module Middleman Contract ArrayOf[Pathname], ArrayOf[Pathname] => Any def update(updated_paths, removed_paths) valid_updates = updated_paths - .map { |p| ::Middleman::Util.path_to_source_file(p, @directory, @type, @options.fetch(:destination_dir, false)) } + .map { |p| @files[p] || path_to_source_file(p, @directory, @type, @options[:destination_dir]) } .select(&method(:valid?)) valid_updates.each do |f| - add_file_to_cache(f) + record_file_change(f) logger.debug "== Change (#{f[:types].inspect}): #{f[:relative_path]}" end @@ -245,11 +245,9 @@ module Middleman valid_updates |= related_updates valid_removes = removed_paths - .lazy .select(&@files.method(:key?)) .map(&@files.method(:[])) .select(&method(:valid?)) - .to_a .each do |f| remove_file_from_cache(f) logger.debug "== Deletion (#{f[:types].inspect}): #{f[:relative_path]}" @@ -262,10 +260,28 @@ module Middleman ]) unless valid_updates.empty? && valid_removes.empty? end + # Convert a path to a file resprentation. + # + # @param [Pathname] path The path. + # @return [Middleman::SourceFile] + Contract Pathname, Pathname, Symbol, Maybe[String] => ::Middleman::SourceFile + def path_to_source_file(path, directory, type, destination_dir) + types = Set.new([type]) + + relative_path = path.relative_path_from(directory) + relative_path = File.join(destination_dir, relative_path) if destination_dir + + ::Middleman::SourceFile.new(Pathname(relative_path), path, directory, types, 0) + end + Contract IsA['Middleman::SourceFile'] => Any - def add_file_to_cache(f) - @files[f[:full_path]] = f - @extensionless_files[strip_extensions(f[:full_path])] = f + def record_file_change(f) + if @files[f[:full_path]] + @files[f[:full_path]][:version] += 1 + else + @files[f[:full_path]] = f + @extensionless_files[strip_extensions(f[:full_path])] = f + end end Contract IsA['Middleman::SourceFile'] => Any diff --git a/middleman-core/lib/middleman-core/template_context.rb b/middleman-core/lib/middleman-core/template_context.rb index 8bc8d9bc..8d7fd784 100644 --- a/middleman-core/lib/middleman-core/template_context.rb +++ b/middleman-core/lib/middleman-core/template_context.rb @@ -110,7 +110,7 @@ module Middleman r = sitemap.find_resource_by_path(source_path) if (r && !r.template?) || (Tilt[partial_file[:full_path]].nil? && partial_file[:full_path].exist?) - File.read(partial_file[:full_path]) + partial_file.read else opts = options.dup locs = opts.delete(:locals) diff --git a/middleman-core/lib/middleman-core/util.rb b/middleman-core/lib/middleman-core/util.rb index b5d9af71..db1238b2 100644 --- a/middleman-core/lib/middleman-core/util.rb +++ b/middleman-core/lib/middleman-core/util.rb @@ -35,20 +35,28 @@ module Middleman # @return [Boolean] Contract Or[String, Pathname] => Bool def binary?(filename) - path = Pathname(filename) - ext = path.extname + @@binary_cache ||= {} - # We hardcode detecting of gzipped SVG files - return true if ext == '.svgz' + return @@binary_cache[filename] if @@binary_cache.key?(filename) - return false if Tilt.registered?(ext.sub('.', '')) + @@binary_cache[filename] = begin + path = Pathname(filename) + ext = path.extname - dot_ext = (ext.to_s[0] == '.') ? ext.dup : ".#{ext}" + # We hardcode detecting of gzipped SVG files + if ext == '.svgz' + true + elsif Tilt.registered?(ext.sub('.', '')) + false + else + dot_ext = (ext.to_s[0] == '.') ? ext.dup : ".#{ext}" - if mime = ::Rack::Mime.mime_type(dot_ext, nil) - !nonbinary_mime?(mime) - else - file_contents_include_binary_bytes?(path.to_s) + if mime = ::Rack::Mime.mime_type(dot_ext, nil) + !nonbinary_mime?(mime) + else + file_contents_include_binary_bytes?(path.to_s) + end + end end end @@ -479,20 +487,6 @@ module Middleman result end - # Convert a path to a file resprentation. - # - # @param [Pathname] path The path. - # @return [Middleman::SourceFile] - Contract Pathname, Pathname, Symbol, Bool => ::Middleman::SourceFile - def path_to_source_file(path, directory, type, destination_dir) - types = Set.new([type]) - - relative_path = path.relative_path_from(directory) - relative_path = File.join(destination_dir, relative_path) if destination_dir - - ::Middleman::SourceFile.new(Pathname(relative_path), path, directory, types) - end - # Finds files which should also be considered to be dirty when # the given file(s) are touched. # diff --git a/middleman-core/lib/middleman-core/util/data.rb b/middleman-core/lib/middleman-core/util/data.rb index 6670f989..246ae2e1 100644 --- a/middleman-core/lib/middleman-core/util/data.rb +++ b/middleman-core/lib/middleman-core/util/data.rb @@ -15,13 +15,14 @@ module Middleman # Get the frontmatter and plain content from a file # @param [String] path # @return [Array] - Contract Pathname, Maybe[Symbol] => [Hash, Maybe[String]] - def parse(full_path, frontmatter_delims, known_type=nil) + Contract IsA['Middleman::SourceFile'], Maybe[Symbol] => [Hash, Maybe[String]] + def parse(file, frontmatter_delims, known_type=nil) + full_path = file[:full_path] return [{}, nil] if ::Middleman::Util.binary?(full_path) # Avoid weird race condition when a file is renamed begin - content = File.read(full_path) + content = file.read rescue EOFError, IOError, Errno::ENOENT return [{}, nil] end