Add system-wide file read cache

This commit is contained in:
Thomas Reynolds 2016-01-20 11:50:25 -08:00
parent 827d5fbb1d
commit ffa662a917
14 changed files with 76 additions and 55 deletions

View file

@ -13,5 +13,3 @@ Feature: Import files
Then I should see 'jQuery' Then I should see 'jQuery'
When I go to "/bower_components2/jquery/dist/jquery.js" When I go to "/bower_components2/jquery/dist/jquery.js"
Then I should see 'jQuery' Then I should see 'jQuery'

View file

@ -15,7 +15,7 @@ class NeighborFrontmatter < ::Middleman::Extension
next unless file next unless file
fmdata = ::Middleman::Util::Data.parse(file[:full_path], app.config[:frontmatter_delims], :yaml).first fmdata = ::Middleman::Util::Data.parse(file, app.config[:frontmatter_delims], :yaml).first
opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type) opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type)
opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options) opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options)
ignored = fmdata.delete(:ignored) ignored = fmdata.delete(:ignored)

View file

@ -26,7 +26,7 @@ class NeighborFrontmatter < ::Middleman::Extension
end end
def apply_neighbor_data(resource, file) def apply_neighbor_data(resource, file)
fmdata = ::Middleman::Util::Data.parse(file[:full_path], app.config[:frontmatter_delims], :yaml).first fmdata = ::Middleman::Util::Data.parse(file, app.config[:frontmatter_delims], :yaml).first
opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type) opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type)
opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options) opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options)
ignored = fmdata.delete(:ignored) ignored = fmdata.delete(:ignored)

View file

@ -100,10 +100,10 @@ module Middleman
basename = File.basename(data_path, extension) basename = File.basename(data_path, extension)
if %w(.yaml .yml).include?(extension) if %w(.yaml .yml).include?(extension)
data, postscript = ::Middleman::Util::Data.parse(file[:full_path], @app.config[:frontmatter_delims], :yaml) data, postscript = ::Middleman::Util::Data.parse(file, @app.config[:frontmatter_delims], :yaml)
data[:postscript] = postscript if !postscript.nil? && data.is_a?(Hash) data[:postscript] = postscript if !postscript.nil? && data.is_a?(Hash)
elsif extension == '.json' elsif extension == '.json'
data, _postscript = ::Middleman::Util::Data.parse(file[:full_path], @app.config[:frontmatter_delims], :json) data, _postscript = ::Middleman::Util::Data.parse(file, @app.config[:frontmatter_delims], :json)
else else
return return
end end

View file

@ -72,8 +72,10 @@ module Middleman::CoreExtensions
return [{}, nil] unless file return [{}, nil] unless file
@cache[file[:full_path]] ||= ::Middleman::Util::Data.parse( return @cache[file[:full_path]] if @cache.key?(file[:full_path])
file[:full_path],
@cache[file[:full_path]] = ::Middleman::Util::Data.parse(
file,
app.config[:frontmatter_delims] app.config[:frontmatter_delims]
) )
end end

View file

@ -97,11 +97,12 @@ module Middleman
Contract String Contract String
def template_data_for_file def template_data_for_file
if @app.extensions[:front_matter] if @app.extensions[:front_matter]
@app.extensions[:front_matter].template_data_for_file(@path) || File.read(@path) result = @app.extensions[:front_matter].template_data_for_file(@path)
else return result unless result.nil?
file = @app.files.find(:source, @path)
file.read if file
end end
file = @app.files.find(:source, @path)
file ? file.read : File.read(@path)
end end
protected protected

View file

@ -235,8 +235,8 @@ module Middleman
if ssl_certificate || ssl_private_key if ssl_certificate || ssl_private_key
raise 'You must provide both :ssl_certificate and :ssl_private_key' unless ssl_private_key && ssl_certificate raise 'You must provide both :ssl_certificate and :ssl_private_key' unless ssl_private_key && ssl_certificate
http_opts[:SSLCertificate] = OpenSSL::X509::Certificate.new File.read ssl_certificate http_opts[:SSLCertificate] = OpenSSL::X509::Certificate.new ::File.read ssl_certificate
http_opts[:SSLPrivateKey] = OpenSSL::PKey::RSA.new File.read ssl_private_key http_opts[:SSLPrivateKey] = OpenSSL::PKey::RSA.new ::File.read ssl_private_key
else else
# use a generated self-signed cert # use a generated self-signed cert
http_opts[:SSLCertName] = [ http_opts[:SSLCertName] = [

View file

@ -14,7 +14,7 @@ module Middleman
def read_template_file(template_path, _) def read_template_file(template_path, _)
file = app.files.find(:source, "_#{template_path}.liquid") file = app.files.find(:source, "_#{template_path}.liquid")
raise ::Liquid::FileSystemError, "No such template '#{template_path}'" unless file raise ::Liquid::FileSystemError, "No such template '#{template_path}'" unless file
File.read(file[:full_path]) file.read
end end
# @return Array<Middleman::Sitemap::Resource> # @return Array<Middleman::Sitemap::Resource>

View file

@ -54,7 +54,7 @@ module Middleman
source = Pathname(source) if source && source.is_a?(String) source = Pathname(source) if source && source.is_a?(String)
@file_descriptor = if source && source.is_a?(Pathname) @file_descriptor = if source && source.is_a?(Pathname)
::Middleman::SourceFile.new(source.relative_path_from(@app.source_dir), source, @app.source_dir, Set.new([:source])) ::Middleman::SourceFile.new(source.relative_path_from(@app.source_dir), source, @app.source_dir, Set.new([:source]), 0)
else else
source source
end end

View file

@ -3,7 +3,12 @@ require 'middleman-core/contracts'
module Middleman module Middleman
# The standard "record" that contains information about a file on disk. # The standard "record" that contains information about a file on disk.
SourceFile = Struct.new :relative_path, :full_path, :directory, :types SourceFile = Struct.new(:relative_path, :full_path, :directory, :types, :version) do
def read
::Middleman::Sources.file_cache[full_path] ||= {}
::Middleman::Sources.file_cache[full_path][version] ||= ::File.read(full_path)
end
end
# Sources handle multiple on-disk collections of files which make up # Sources handle multiple on-disk collections of files which make up
# a Middleman project. They are separated by `type` which can then be # a Middleman project. They are separated by `type` which can then be
@ -36,6 +41,8 @@ module Middleman
# Reference to the global logger. # Reference to the global logger.
def_delegator :@app, :logger def_delegator :@app, :logger
cattr_accessor :file_cache
# Built-in types # Built-in types
# :source, :data, :locales, :reload # :source, :data, :locales, :reload
@ -50,6 +57,8 @@ module Middleman
@watchers = watchers @watchers = watchers
@sorted_watchers = @watchers.dup.freeze @sorted_watchers = @watchers.dup.freeze
::Middleman::Sources.file_cache = {}
@options = options @options = options
# Set of procs wanting to be notified of changes # Set of procs wanting to be notified of changes
@ -177,9 +186,10 @@ module Middleman
# @return [Middleman::SourceFile, nil] # @return [Middleman::SourceFile, nil]
Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String, Maybe[Bool] => Maybe[SourceFile] Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String, Maybe[Bool] => Maybe[SourceFile]
def find(types, path, glob=false) def find(types, path, glob=false)
array_of_types = Array(types)
watchers watchers
.lazy .select { |d| array_of_types.include?(d.type) }
.select { |d| Array(types).include?(d.type) }
.map { |d| d.find(path, glob) } .map { |d| d.find(path, glob) }
.reject(&:nil?) .reject(&:nil?)
.first .first
@ -193,7 +203,6 @@ module Middleman
Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String => Bool Contract Or[Symbol, ArrayOf[Symbol], SetOf[Symbol]], String => Bool
def exists?(types, path) def exists?(types, path)
watchers watchers
.lazy
.select { |d| Array(types).include?(d.type) } .select { |d| Array(types).include?(d.type) }
.any? { |d| d.exists?(path) } .any? { |d| d.exists?(path) }
end end

View file

@ -228,11 +228,11 @@ module Middleman
Contract ArrayOf[Pathname], ArrayOf[Pathname] => Any Contract ArrayOf[Pathname], ArrayOf[Pathname] => Any
def update(updated_paths, removed_paths) def update(updated_paths, removed_paths)
valid_updates = updated_paths valid_updates = updated_paths
.map { |p| ::Middleman::Util.path_to_source_file(p, @directory, @type, @options.fetch(:destination_dir, false)) } .map { |p| @files[p] || path_to_source_file(p, @directory, @type, @options[:destination_dir]) }
.select(&method(:valid?)) .select(&method(:valid?))
valid_updates.each do |f| valid_updates.each do |f|
add_file_to_cache(f) record_file_change(f)
logger.debug "== Change (#{f[:types].inspect}): #{f[:relative_path]}" logger.debug "== Change (#{f[:types].inspect}): #{f[:relative_path]}"
end end
@ -245,11 +245,9 @@ module Middleman
valid_updates |= related_updates valid_updates |= related_updates
valid_removes = removed_paths valid_removes = removed_paths
.lazy
.select(&@files.method(:key?)) .select(&@files.method(:key?))
.map(&@files.method(:[])) .map(&@files.method(:[]))
.select(&method(:valid?)) .select(&method(:valid?))
.to_a
.each do |f| .each do |f|
remove_file_from_cache(f) remove_file_from_cache(f)
logger.debug "== Deletion (#{f[:types].inspect}): #{f[:relative_path]}" logger.debug "== Deletion (#{f[:types].inspect}): #{f[:relative_path]}"
@ -262,10 +260,28 @@ module Middleman
]) unless valid_updates.empty? && valid_removes.empty? ]) unless valid_updates.empty? && valid_removes.empty?
end end
# Convert a path to a file resprentation.
#
# @param [Pathname] path The path.
# @return [Middleman::SourceFile]
Contract Pathname, Pathname, Symbol, Maybe[String] => ::Middleman::SourceFile
def path_to_source_file(path, directory, type, destination_dir)
types = Set.new([type])
relative_path = path.relative_path_from(directory)
relative_path = File.join(destination_dir, relative_path) if destination_dir
::Middleman::SourceFile.new(Pathname(relative_path), path, directory, types, 0)
end
Contract IsA['Middleman::SourceFile'] => Any Contract IsA['Middleman::SourceFile'] => Any
def add_file_to_cache(f) def record_file_change(f)
@files[f[:full_path]] = f if @files[f[:full_path]]
@extensionless_files[strip_extensions(f[:full_path])] = f @files[f[:full_path]][:version] += 1
else
@files[f[:full_path]] = f
@extensionless_files[strip_extensions(f[:full_path])] = f
end
end end
Contract IsA['Middleman::SourceFile'] => Any Contract IsA['Middleman::SourceFile'] => Any

View file

@ -110,7 +110,7 @@ module Middleman
r = sitemap.find_resource_by_path(source_path) r = sitemap.find_resource_by_path(source_path)
if (r && !r.template?) || (Tilt[partial_file[:full_path]].nil? && partial_file[:full_path].exist?) if (r && !r.template?) || (Tilt[partial_file[:full_path]].nil? && partial_file[:full_path].exist?)
File.read(partial_file[:full_path]) partial_file.read
else else
opts = options.dup opts = options.dup
locs = opts.delete(:locals) locs = opts.delete(:locals)

View file

@ -35,20 +35,28 @@ module Middleman
# @return [Boolean] # @return [Boolean]
Contract Or[String, Pathname] => Bool Contract Or[String, Pathname] => Bool
def binary?(filename) def binary?(filename)
path = Pathname(filename) @@binary_cache ||= {}
ext = path.extname
# We hardcode detecting of gzipped SVG files return @@binary_cache[filename] if @@binary_cache.key?(filename)
return true if ext == '.svgz'
return false if Tilt.registered?(ext.sub('.', '')) @@binary_cache[filename] = begin
path = Pathname(filename)
ext = path.extname
dot_ext = (ext.to_s[0] == '.') ? ext.dup : ".#{ext}" # We hardcode detecting of gzipped SVG files
if ext == '.svgz'
true
elsif Tilt.registered?(ext.sub('.', ''))
false
else
dot_ext = (ext.to_s[0] == '.') ? ext.dup : ".#{ext}"
if mime = ::Rack::Mime.mime_type(dot_ext, nil) if mime = ::Rack::Mime.mime_type(dot_ext, nil)
!nonbinary_mime?(mime) !nonbinary_mime?(mime)
else else
file_contents_include_binary_bytes?(path.to_s) file_contents_include_binary_bytes?(path.to_s)
end
end
end end
end end
@ -479,20 +487,6 @@ module Middleman
result result
end end
# Convert a path to a file resprentation.
#
# @param [Pathname] path The path.
# @return [Middleman::SourceFile]
Contract Pathname, Pathname, Symbol, Bool => ::Middleman::SourceFile
def path_to_source_file(path, directory, type, destination_dir)
types = Set.new([type])
relative_path = path.relative_path_from(directory)
relative_path = File.join(destination_dir, relative_path) if destination_dir
::Middleman::SourceFile.new(Pathname(relative_path), path, directory, types)
end
# Finds files which should also be considered to be dirty when # Finds files which should also be considered to be dirty when
# the given file(s) are touched. # the given file(s) are touched.
# #

View file

@ -15,13 +15,14 @@ module Middleman
# Get the frontmatter and plain content from a file # Get the frontmatter and plain content from a file
# @param [String] path # @param [String] path
# @return [Array<Hash, String>] # @return [Array<Hash, String>]
Contract Pathname, Maybe[Symbol] => [Hash, Maybe[String]] Contract IsA['Middleman::SourceFile'], Maybe[Symbol] => [Hash, Maybe[String]]
def parse(full_path, frontmatter_delims, known_type=nil) def parse(file, frontmatter_delims, known_type=nil)
full_path = file[:full_path]
return [{}, nil] if ::Middleman::Util.binary?(full_path) return [{}, nil] if ::Middleman::Util.binary?(full_path)
# Avoid weird race condition when a file is renamed # Avoid weird race condition when a file is renamed
begin begin
content = File.read(full_path) content = file.read
rescue EOFError, IOError, Errno::ENOENT rescue EOFError, IOError, Errno::ENOENT
return [{}, nil] return [{}, nil]
end end