Add system-wide file read cache

middleman-core/features/import_files.feature

@@ -13,5 +13,3 @@ Feature: Import files
     Then I should see 'jQuery'
     When I go to "/bower_components2/jquery/dist/jquery.js"
     Then I should see 'jQuery'
-
-

middleman-core/fixtures/frontmatter-neighbor-app/config.rb

@@ -15,7 +15,7 @@ class NeighborFrontmatter < ::Middleman::Extension
 
       next unless file
 
-      fmdata = ::Middleman::Util::Data.parse(file[:full_path], app.config[:frontmatter_delims], :yaml).first
+      fmdata = ::Middleman::Util::Data.parse(file, app.config[:frontmatter_delims], :yaml).first
       opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type)
       opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options)
       ignored = fmdata.delete(:ignored)

middleman-core/fixtures/frontmatter-settings-neighbor-app/config.rb

@@ -26,7 +26,7 @@ class NeighborFrontmatter < ::Middleman::Extension
   end
 
   def apply_neighbor_data(resource, file)
-    fmdata = ::Middleman::Util::Data.parse(file[:full_path], app.config[:frontmatter_delims], :yaml).first
+    fmdata = ::Middleman::Util::Data.parse(file, app.config[:frontmatter_delims], :yaml).first
     opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type)
     opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options)
     ignored = fmdata.delete(:ignored)

middleman-core/lib/middleman-core/core_extensions/data.rb

@@ -100,10 +100,10 @@ module Middleman
           basename  = File.basename(data_path, extension)
 
           if %w(.yaml .yml).include?(extension)
-            data, postscript = ::Middleman::Util::Data.parse(file[:full_path], @app.config[:frontmatter_delims], :yaml)
+            data, postscript = ::Middleman::Util::Data.parse(file, @app.config[:frontmatter_delims], :yaml)
             data[:postscript] = postscript if !postscript.nil? && data.is_a?(Hash)
           elsif extension == '.json'
-            data, _postscript = ::Middleman::Util::Data.parse(file[:full_path], @app.config[:frontmatter_delims], :json)
+            data, _postscript = ::Middleman::Util::Data.parse(file, @app.config[:frontmatter_delims], :json)
           else
             return
           end

middleman-core/lib/middleman-core/core_extensions/front_matter.rb

@@ -72,8 +72,10 @@ module Middleman::CoreExtensions
 
       return [{}, nil] unless file
 
-      @cache[file[:full_path]] ||= ::Middleman::Util::Data.parse(
+      return @cache[file[:full_path]] if @cache.key?(file[:full_path])
-        file[:full_path],
+
+      @cache[file[:full_path]] = ::Middleman::Util::Data.parse(
+        file,
         app.config[:frontmatter_delims]
       )
     end

middleman-core/lib/middleman-core/file_renderer.rb

@@ -97,11 +97,12 @@ module Middleman
     Contract String
     def template_data_for_file
       if @app.extensions[:front_matter]
-        @app.extensions[:front_matter].template_data_for_file(@path) || File.read(@path)
+        result = @app.extensions[:front_matter].template_data_for_file(@path)
-      else
+        return result unless result.nil?
-        file = @app.files.find(:source, @path)
-        file.read if file
       end
+
+      file = @app.files.find(:source, @path)
+      file ? file.read : File.read(@path)
     end
 
     protected

middleman-core/lib/middleman-core/preview_server.rb

@@ -235,8 +235,8 @@ module Middleman
 
           if ssl_certificate || ssl_private_key
             raise 'You must provide both :ssl_certificate and :ssl_private_key' unless ssl_private_key && ssl_certificate
-            http_opts[:SSLCertificate] = OpenSSL::X509::Certificate.new File.read ssl_certificate
+            http_opts[:SSLCertificate] = OpenSSL::X509::Certificate.new ::File.read ssl_certificate
-            http_opts[:SSLPrivateKey] = OpenSSL::PKey::RSA.new File.read ssl_private_key
+            http_opts[:SSLPrivateKey] = OpenSSL::PKey::RSA.new ::File.read ssl_private_key
           else
             # use a generated self-signed cert
             http_opts[:SSLCertName] = [

middleman-core/lib/middleman-core/renderers/liquid.rb

@@ -14,7 +14,7 @@ module Middleman
       def read_template_file(template_path, _)
         file = app.files.find(:source, "_#{template_path}.liquid")
         raise ::Liquid::FileSystemError, "No such template '#{template_path}'" unless file
-        File.read(file[:full_path])
+        file.read
       end
 
       # @return Array<Middleman::Sitemap::Resource>

middleman-core/lib/middleman-core/sitemap/resource.rb

@@ -54,7 +54,7 @@ module Middleman
         source = Pathname(source) if source && source.is_a?(String)
 
         @file_descriptor = if source && source.is_a?(Pathname)
-          ::Middleman::SourceFile.new(source.relative_path_from(@app.source_dir), source, @app.source_dir, Set.new([:source]))
+          ::Middleman::SourceFile.new(source.relative_path_from(@app.source_dir), source, @app.source_dir, Set.new([:source]), 0)
         else
           source
         end

middleman-core/lib/middleman-core/sources.rb

@@ -3,7 +3,12 @@ require 'middleman-core/contracts'
 
 module Middleman
   # The standard "record" that contains information about a file on disk.
-  SourceFile = Struct.new :relative_path, :full_path, :directory, :types
+  SourceFile = Struct.new(:relative_path, :full_path, :directory, :types, :version) do
+    def read
+      ::Middleman::Sources.file_cache[full_path] ||= {}
+      ::Middleman::Sources.file_cache[full_path][version] ||= ::File.read(full_path)
+    end
+  end
 
   # Sources handle multiple on-disk collections of files which make up
   # a Middleman project. They are separated by `type` which can then be
@@ -36,6 +41,8 @@ module Middleman
     # Reference to the global logger.
     def_delegator :@app, :logger
 
+    cattr_accessor :file_cache
+
     # Built-in types
     # :source, :data, :locales, :reload
 
@@ -50,6 +57,8 @@ module Middleman
       @watchers = watchers
       @sorted_watchers = @watchers.dup.freeze
 
+      ::Middleman::Sources.file_cache = {}
+
       @options = options
 
       # Set of procs wanting to be notified of changes
@@ -177,9 +186,10 @@ module Middleman
     # @return [Middleman::SourceFile, nil]
     Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String, Maybe[Bool] => Maybe[SourceFile]
     def find(types, path, glob=false)
+      array_of_types = Array(types)
+
       watchers
-        .lazy
+        .select { |d| array_of_types.include?(d.type) }
-        .select { |d| Array(types).include?(d.type) }
         .map { |d| d.find(path, glob) }
         .reject(&:nil?)
         .first
@@ -193,7 +203,6 @@ module Middleman
     Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String => Bool
     def exists?(types, path)
       watchers
-        .lazy
         .select { |d| Array(types).include?(d.type) }
         .any? { |d| d.exists?(path) }
     end

middleman-core/lib/middleman-core/sources/source_watcher.rb

@@ -228,11 +228,11 @@ module Middleman
     Contract ArrayOf[Pathname], ArrayOf[Pathname] => Any
     def update(updated_paths, removed_paths)
       valid_updates = updated_paths
-                      .map { |p| ::Middleman::Util.path_to_source_file(p, @directory, @type, @options.fetch(:destination_dir, false)) }
+                      .map { |p| @files[p] || path_to_source_file(p, @directory, @type, @options[:destination_dir]) }
                       .select(&method(:valid?))
 
       valid_updates.each do |f|
-        add_file_to_cache(f)
+        record_file_change(f)
         logger.debug "== Change (#{f[:types].inspect}): #{f[:relative_path]}"
       end
 
@@ -245,11 +245,9 @@ module Middleman
       valid_updates |= related_updates
 
       valid_removes = removed_paths
-                      .lazy
                       .select(&@files.method(:key?))
                       .map(&@files.method(:[]))
                       .select(&method(:valid?))
-                      .to_a
                       .each do |f|
                         remove_file_from_cache(f)
                         logger.debug "== Deletion (#{f[:types].inspect}): #{f[:relative_path]}"
@@ -262,10 +260,28 @@ module Middleman
                         ]) unless valid_updates.empty? && valid_removes.empty?
     end
 
+    # Convert a path to a file resprentation.
+    #
+    # @param [Pathname] path The path.
+    # @return [Middleman::SourceFile]
+    Contract Pathname, Pathname, Symbol, Maybe[String] => ::Middleman::SourceFile
+    def path_to_source_file(path, directory, type, destination_dir)
+      types = Set.new([type])
+
+      relative_path = path.relative_path_from(directory)
+      relative_path = File.join(destination_dir, relative_path) if destination_dir
+
+      ::Middleman::SourceFile.new(Pathname(relative_path), path, directory, types, 0)
+    end
+
     Contract IsA['Middleman::SourceFile'] => Any
-    def add_file_to_cache(f)
+    def record_file_change(f)
-      @files[f[:full_path]] = f
+      if @files[f[:full_path]]
-      @extensionless_files[strip_extensions(f[:full_path])] = f
+        @files[f[:full_path]][:version] += 1
+      else
+        @files[f[:full_path]] = f
+        @extensionless_files[strip_extensions(f[:full_path])] = f
+      end
     end
 
     Contract IsA['Middleman::SourceFile'] => Any

middleman-core/lib/middleman-core/template_context.rb

@@ -110,7 +110,7 @@ module Middleman
       r = sitemap.find_resource_by_path(source_path)
 
       if (r && !r.template?) || (Tilt[partial_file[:full_path]].nil? && partial_file[:full_path].exist?)
-        File.read(partial_file[:full_path])
+        partial_file.read
       else
         opts = options.dup
         locs = opts.delete(:locals)

middleman-core/lib/middleman-core/util.rb

@@ -35,20 +35,28 @@ module Middleman
     # @return [Boolean]
     Contract Or[String, Pathname] => Bool
     def binary?(filename)
-      path = Pathname(filename)
+      @@binary_cache ||= {}
-      ext = path.extname
 
-      # We hardcode detecting of gzipped SVG files
+      return @@binary_cache[filename] if @@binary_cache.key?(filename)
-      return true if ext == '.svgz'
 
-      return false if Tilt.registered?(ext.sub('.', ''))
+      @@binary_cache[filename] = begin
+        path = Pathname(filename)
+        ext = path.extname
 
-      dot_ext = (ext.to_s[0] == '.') ? ext.dup : ".#{ext}"
+        # We hardcode detecting of gzipped SVG files
+        if ext == '.svgz'
+          true
+        elsif Tilt.registered?(ext.sub('.', ''))
+          false
+        else
+          dot_ext = (ext.to_s[0] == '.') ? ext.dup : ".#{ext}"
 
-      if mime = ::Rack::Mime.mime_type(dot_ext, nil)
+          if mime = ::Rack::Mime.mime_type(dot_ext, nil)
-        !nonbinary_mime?(mime)
+            !nonbinary_mime?(mime)
-      else
+          else
-        file_contents_include_binary_bytes?(path.to_s)
+            file_contents_include_binary_bytes?(path.to_s)
+          end
+        end
       end
     end
 
@@ -479,20 +487,6 @@ module Middleman
       result
     end
 
-    # Convert a path to a file resprentation.
-    #
-    # @param [Pathname] path The path.
-    # @return [Middleman::SourceFile]
-    Contract Pathname, Pathname, Symbol, Bool => ::Middleman::SourceFile
-    def path_to_source_file(path, directory, type, destination_dir)
-      types = Set.new([type])
-
-      relative_path = path.relative_path_from(directory)
-      relative_path = File.join(destination_dir, relative_path) if destination_dir
-
-      ::Middleman::SourceFile.new(Pathname(relative_path), path, directory, types)
-    end
-
     # Finds files which should also be considered to be dirty when
     # the given file(s) are touched.
     #

middleman-core/lib/middleman-core/util/data.rb

@@ -15,13 +15,14 @@ module Middleman
       # Get the frontmatter and plain content from a file
       # @param [String] path
       # @return [Array<Hash, String>]
-      Contract Pathname, Maybe[Symbol] => [Hash, Maybe[String]]
+      Contract IsA['Middleman::SourceFile'], Maybe[Symbol] => [Hash, Maybe[String]]
-      def parse(full_path, frontmatter_delims, known_type=nil)
+      def parse(file, frontmatter_delims, known_type=nil)
+        full_path = file[:full_path]
         return [{}, nil] if ::Middleman::Util.binary?(full_path)
 
         # Avoid weird race condition when a file is renamed
         begin
-          content = File.read(full_path)
+          content = file.read
         rescue EOFError, IOError, Errno::ENOENT
           return [{}, nil]
         end