Merge pull request #1322 from middleman/sources

Multiple Source watchers

Gemfile

@@ -6,6 +6,8 @@ gem 'yard', '~> 0.8', require: false
 
 # Test tools
 gem 'pry', '~> 0.10', group: :development, require: false
+gem 'pry-debugger', platforms: [:ruby_19, :ruby_20], require: false
+gem 'pry-stack_explorer', platforms: [:ruby_19, :ruby_20], require: false
 gem 'aruba', '~> 0.6', require: false
 gem 'rspec', '~> 3.0', require: false
 gem 'fivemat', '~> 1.3', require: false

middleman-cli/lib/middleman-cli/server.rb

@@ -36,10 +36,6 @@ module Middleman::Cli
                   type: :boolean,
                   default: false,
                   desc: 'Generate profiling report for server startup'
-    method_option :reload_paths,
-                  type: :string,
-                  default: false,
-                  desc: 'Additional paths to auto-reload when files change'
     method_option :force_polling,
                   type: :boolean,
                   default: false,

middleman-core/features/multiple-sources.feature

@@ -0,0 +1,27 @@
+Feature: Allow multiple sources to be setup.
+
+  Scenario: Three source directories.
+    Given the Server is running at "multiple-sources-app"
+    When I go to "/index.html"
+    Then I should see "Default Source"
+
+    When I go to "/index1.html"
+    Then I should see "Source 1"
+
+    When I go to "/index2.html"
+    Then I should see "Source 2"
+
+    When I go to "/override-in-two.html"
+    Then I should see "Overridden 2"
+
+    When I go to "/override-in-one.html"
+    Then I should see "Opposite 2"
+
+  Scenario: Three data directories.
+    Given the Server is running at "multiple-data-sources-app"
+    When I go to "/index.html"
+    Then I should see "Default: Data Default"
+    Then I should see "Data 1: Data 1"
+    Then I should see "Data 2: Data 2"
+    Then I should see "Override in Two: Overridden 2"
+    Then I should see "Override in One: Opposite 2"

middleman-core/fixtures/frontmatter-neighbor-app/config.rb

@@ -9,9 +9,13 @@ class NeighborFrontmatter < ::Middleman::Extension
     resources.each do |resource|
       next unless resource.source_file
 
-      neighbor = "#{resource.source_file}.frontmatter"
+      neighbor = "#{resource.source_file[:relative_path]}.frontmatter"
-      if File.exists?(neighbor)
+
-        fmdata = app.extensions[:front_matter].frontmatter_and_content(neighbor).first
+      file = app.files.find(:source, neighbor)
+
+      next unless file
+
+      fmdata = app.extensions[:front_matter].frontmatter_and_content(file[:full_path]).first
       opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type)
       opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options)
       ignored = fmdata.delete(:ignored)
@@ -19,7 +23,6 @@ class NeighborFrontmatter < ::Middleman::Extension
       resource.ignore! if ignored == true && !resource.is_a?(::Middleman::Sitemap::ProxyResource)
     end
   end
-  end
 end
 
 Middleman::Extensions.register :neighbor_frontmatter, NeighborFrontmatter unless Middleman::Extensions.registered.include? :neighbor_frontmatter

middleman-core/fixtures/frontmatter-settings-neighbor-app/config.rb

@@ -14,9 +14,13 @@ class NeighborFrontmatter < ::Middleman::Extension
     resources.each do |resource|
       next unless resource.source_file
 
-      neighbor = "#{resource.source_file}.frontmatter"
+      neighbor = "#{resource.source_file[:relative_path]}.frontmatter"
-      if File.exists?(neighbor)
+
-        fmdata = app.extensions[:front_matter].frontmatter_and_content(neighbor).first
+      file = app.files.find(:source, neighbor)
+
+      next unless file
+
+      fmdata = app.extensions[:front_matter].frontmatter_and_content(file[:full_path]).first
       opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type)
       opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options)
       ignored = fmdata.delete(:ignored)
@@ -24,7 +28,6 @@ class NeighborFrontmatter < ::Middleman::Extension
       resource.ignore! if ignored == true && !resource.is_a?(::Middleman::Sitemap::ProxyResource)
     end
   end
-  end
 end
 
 Middleman::Extensions.register :neighbor_frontmatter, NeighborFrontmatter unless Middleman::Extensions.registered.include? :neighbor_frontmatter

middleman-core/fixtures/more-traversal-app/source/layout.erb

@@ -1,6 +1,6 @@
 Path: <%= current_page.path %>
 
-Source: <%= current_page.source_file.sub(root + "/", "") %>
+Source: <%= current_page.source_file[:full_path].sub(root + "/", "") %>
 
 <% if current_page.parent %>
   Parent: <%= current_page.parent.path %>

middleman-core/fixtures/multiple-data-sources-app/config.rb

@@ -0,0 +1,3 @@
+files.watch :data, path: File.join(root, 'data0'), priority: 100
+files.watch :data, path: File.join(root, 'data1')
+files.watch :data, path: File.join(root, 'data2')

middleman-core/fixtures/multiple-data-sources-app/data/data.yml

@@ -0,0 +1 @@
+title: Data Default

middleman-core/fixtures/multiple-data-sources-app/data/two.yml

@@ -0,0 +1 @@
+title: Overridden Default

middleman-core/fixtures/multiple-data-sources-app/data0/one.yml

@@ -0,0 +1 @@
+title: Opposite 2

middleman-core/fixtures/multiple-data-sources-app/data1/data1.yml

@@ -0,0 +1 @@
+title: Data 1

middleman-core/fixtures/multiple-data-sources-app/data1/one.yml

@@ -0,0 +1 @@
+title: Opposite 1

middleman-core/fixtures/multiple-data-sources-app/data2/data2.yml

@@ -0,0 +1 @@
+title: Data 2

middleman-core/fixtures/multiple-data-sources-app/data2/two.yml

@@ -0,0 +1 @@
+title: Overridden 2

middleman-core/fixtures/multiple-data-sources-app/source/index.html.erb

@@ -0,0 +1,5 @@
+Default: <%= data.data.title %>
+Data 1: <%= data.data1.title %>
+Data 2: <%= data.data2.title %>
+Override in Two: <%= data.two.title %>
+Override in One: <%= data.one.title %>

middleman-core/fixtures/multiple-sources-app/config.rb

@@ -0,0 +1,3 @@
+files.watch :source, path: File.join(root, 'source0'), priority: 100
+files.watch :source, path: File.join(root, 'source1')
+files.watch :source, path: File.join(root, 'source2')

middleman-core/fixtures/multiple-sources-app/source/index.html.erb

@@ -0,0 +1 @@
+Default Source

middleman-core/fixtures/multiple-sources-app/source/override-in-two.html.erb

@@ -0,0 +1 @@
+Overridden Default

middleman-core/fixtures/multiple-sources-app/source0/override-in-one.html.erb

@@ -0,0 +1 @@
+Opposite 2

middleman-core/fixtures/multiple-sources-app/source1/index1.html.erb

@@ -0,0 +1 @@
+Source 1

middleman-core/fixtures/multiple-sources-app/source1/override-in-one.html.erb

@@ -0,0 +1 @@
+Opposite 1

middleman-core/fixtures/multiple-sources-app/source2/index2.html.erb

@@ -0,0 +1 @@
+Source 2

middleman-core/fixtures/multiple-sources-app/source2/override-in-two.html.erb

@@ -0,0 +1 @@
+Overridden 2

middleman-core/fixtures/traversal-app/source/layout.erb

@@ -1,6 +1,6 @@
 Path: <%= current_page.path %>
 
-Source: <%= current_page.source_file.sub(root + "/", "") %>
+Source: <%= current_page.source_file[:full_path].sub(root + "/", "") %>
 
 <% if current_page.parent %>
   Parent: <%= current_page.parent.path %>

middleman-core/lib/middleman-core/application.rb

@@ -73,6 +73,8 @@ module Middleman
     # Runs after the build is finished
     define_hook :after_build
 
+    define_hook :before_shutdown
+
     define_hook :before_render
     define_hook :after_render
 
@@ -145,22 +147,19 @@ module Middleman
 
     # Setup callbacks which can exclude paths from the sitemap
     config.define_setting :ignored_sitemap_matchers, {
-      # dotfiles and folders in the root
-      root_dotfiles: proc { |file| file.start_with?('.') },
-
-      # Files starting with an dot, but not .htaccess
-      source_dotfiles: proc { |file|
-        file =~ %r{/\.} && file !~ %r{/\.(htaccess|htpasswd|nojekyll)}
-      },
-
       # Files starting with an underscore, but not a double-underscore
-      partials: proc { |file| file =~ %r{/_[^_]} },
+      partials: proc { |file| File.basename(file[:relative_path]).match %r{^_[^_]} },
 
-      layout: proc { |file, sitemap_app|
+      layout: proc { |file, _sitemap_app|
-        file.start_with?(File.join(sitemap_app.config[:source], 'layout.')) || file.start_with?(File.join(sitemap_app.config[:source], 'layouts/'))
+        file[:relative_path].to_s.start_with?('layout.') ||
+        file[:relative_path].to_s.start_with?('layouts/')
       }
     }, 'Callbacks that can exclude paths from the sitemap'
 
+    config.define_setting :watcher_disable, false, 'If the Listen watcher should not run'
+    config.define_setting :watcher_force_polling, false, 'If the Listen watcher should run in polling mode'
+    config.define_setting :watcher_latency, nil, 'The Listen watcher latency'
+
     attr_reader :config_context
     attr_reader :sitemap
     attr_reader :cache
@@ -168,6 +167,7 @@ module Middleman
     attr_reader :config
     attr_reader :generic_template_context
     attr_reader :extensions
+    attr_reader :sources
 
     Contract None => SetOf['Middleman::Application::MiddlewareDescriptor']
     attr_reader :middleware
@@ -200,7 +200,13 @@ module Middleman
       @config = ::Middleman::Configuration::ConfigurationManager.new
       @config.load_settings(self.class.config.all_settings)
 
+      config[:source] = ENV['MM_SOURCE'] if ENV['MM_SOURCE']
+
       @extensions = ::Middleman::ExtensionManager.new(self)
+
+      # Evaluate a passed block if given
+      config_context.instance_exec(&block) if block_given?
+
       @extensions.auto_activate(:before_sitemap)
 
       # Initialize the Sitemap
@@ -211,8 +217,6 @@ module Middleman
         Encoding.default_external = config[:encoding]
       end
 
-      config[:source] = ENV['MM_SOURCE'] if ENV['MM_SOURCE']
-
       ::Middleman::Extension.clear_after_extension_callbacks
 
       @extensions.auto_activate(:before_configuration)
@@ -221,7 +225,7 @@ module Middleman
 
       run_hook :before_configuration
 
-      evaluate_configuration(&block)
+      evaluate_configuration
 
       # This is for making the tests work - since the tests
       # don't completely reload middleman, I18n.load_path can get
@@ -250,10 +254,7 @@ module Middleman
       @config_context.execute_ready_callbacks
     end
 
-    def evaluate_configuration(&block)
+    def evaluate_configuration
-      # Evaluate a passed block if given
-      config_context.instance_exec(&block) if block_given?
-
       # Check for and evaluate local configuration in `config.rb`
       local_config = File.join(root, 'config.rb')
       if File.exist? local_config
@@ -296,13 +297,6 @@ module Middleman
       config[:environment] == key
     end
 
-    # The full path to the source directory
-    #
-    # @return [String]
-    def source_dir
-      File.join(root, config[:source])
-    end
-
     MiddlewareDescriptor = Struct.new(:class, :options, :block)
 
     # Use Rack middleware
@@ -325,6 +319,10 @@ module Middleman
       @mappings << MapDescriptor.new(map, block)
     end
 
+    def shutdown!
+      run_hook :before_shutdown
+    end
+
     # Work around this bug: http://bugs.ruby-lang.org/issues/4521
     # where Ruby will call to_s/inspect while printing exception
     # messages, which can take a long time (minutes at full CPU)

middleman-core/lib/middleman-core/builder.rb

@@ -23,7 +23,7 @@ module Middleman
     # @param [Hash] opts The builder options
     def initialize(app, opts={})
       @app = app
-      @source_dir = Pathname(@app.source_dir)
+      @source_dir = Pathname(File.join(@app.root, @app.config[:source]))
       @build_dir = Pathname(@app.config[:build_dir])
 
       if @build_dir.expand_path.relative_path_from(@source_dir).to_s =~ /\A[.\/]+\Z/
@@ -83,7 +83,7 @@ module Middleman
       logger.debug '== Checking for Compass sprites'
 
       # Double-check for compass sprites
-      @app.files.find_new_files((@source_dir + @app.config[:images_dir]).relative_path_from(@app.root_path))
+      @app.files.find_new_files!
       @app.sitemap.ensure_resource_list_updated!
 
       css_files
@@ -170,7 +170,7 @@ module Middleman
 
       begin
         if resource.binary?
-          export_file!(output_file, Pathname(resource.source_file))
+          export_file!(output_file, resource.source_file[:full_path])
         else
           response = @rack.get(URI.escape(resource.request_path))
 

middleman-core/lib/middleman-core/core_extensions/data.rb

@@ -9,26 +9,37 @@ module Middleman
     class Data < Extension
       attr_reader :data_store
 
-      def before_configuration
+      def initialize(app, config={}, &block)
-        @data_store = DataStore.new(app)
+        super
-        app.config.define_setting :data_dir, 'data', 'The directory data files are stored in'
-
-        app.add_to_config_context :data, &method(:data_store)
 
         # The regex which tells Middleman which files are for data
-        data_file_matcher = /#{app.config[:data_dir]}\/(.*?)[\w-]+\.(yml|yaml|json)$/
+        data_file_matcher = /^(.*?)[\w-]+\.(yml|yaml|json)$/
+
+        @data_store = DataStore.new(app, data_file_matcher)
+        app.config.define_setting :data_dir, 'data', 'The directory data files are stored in'
+
+        app.add_to_config_context(:data, &method(:data_store))
+
+        start_watching(app.config[:data_dir])
+      end
+
+      def start_watching(dir)
+        @original_data_dir = dir
+
+        # Tell the file watcher to observe the :data_dir
+        @watcher = app.files.watch :data,
+                                   path: File.join(app.root, dir),
+                                   ignore: proc { |f| !data_file_matcher.match(f[:relative_path]) }
 
         # Setup data files before anything else so they are available when
         # parsing config.rb
-        app.files.changed(data_file_matcher, &app.extensions[:data].data_store.method(:touch_file))
+        app.files.changed(:data, &@data_store.method(:update_files))
-        app.files.deleted(data_file_matcher, &app.extensions[:data].data_store.method(:remove_file))
-
-        # Tell the file watcher to observe the :data_dir
-        app.files.watch :data do |path, _app|
-          path.match data_file_matcher
       end
 
-        app.files.reload_path(app.config[:data_dir])
+      def after_configuration
+        return unless @original_data_dir != app.config[:data_dir]
+
+        @watcher.update_path(app.config[:data_dir])
       end
 
       helpers do
@@ -44,8 +55,9 @@ module Middleman
         # Setup data store
         #
         # @param [Middleman::Application] app The current instance of Middleman
-        def initialize(app)
+        def initialize(app, data_file_matcher)
           @app = app
+          @data_file_matcher = data_file_matcher
           @local_data = {}
           @local_sources = {}
           @callback_sources = {}
@@ -73,22 +85,26 @@ module Middleman
           @callback_sources
         end
 
+        Contract ArrayOf[IsA['Middleman::SourceFile']], ArrayOf[IsA['Middleman::SourceFile']] => Any
+        def update_files(updated_files, removed_files)
+          updated_files.each(&method(:touch_file))
+          removed_files.each(&method(:remove_file))
+        end
+
         # Update the internal cache for a given file path
         #
         # @param [String] file The file to be re-parsed
         # @return [void]
+        Contract IsA['Middleman::SourceFile'] => Any
         def touch_file(file)
-          root = Pathname(@app.root)
+          data_path = file[:relative_path]
-          full_path = root + file
+          extension = File.extname(data_path)
-          extension = File.extname(file)
+          basename  = File.basename(data_path, extension)
-          basename  = File.basename(file, extension)
-
-          data_path = full_path.relative_path_from(root + @app.config[:data_dir])
 
           if %w(.yaml .yml).include?(extension)
-            data = YAML.load_file(full_path)
+            data = YAML.load_file(file[:full_path])
           elsif extension == '.json'
-            data = ActiveSupport::JSON.decode(full_path.read)
+            data = ActiveSupport::JSON.decode(file[:full_path].read)
           else
             return
           end
@@ -108,13 +124,11 @@ module Middleman
         #
         # @param [String] file The file to be cleared
         # @return [void]
+        Contract IsA['Middleman::SourceFile'] => Any
         def remove_file(file)
-          root = Pathname(@app.root)
+          data_path = file[:relative_path]
-          full_path = root + file
+          extension = File.extname(data_path)
-          extension = File.extname(file)
+          basename  = File.basename(data_path, extension)
-          basename  = File.basename(file, extension)
-
-          data_path = full_path.relative_path_from(root + @app.config[:data_dir])
 
           data_branch = @local_data
 

middleman-core/lib/middleman-core/core_extensions/file_watcher.rb

@@ -1,205 +1,75 @@
-require 'pathname'
-require 'set'
 require 'middleman-core/contracts'
+require 'middleman-core/sources'
 
 module Middleman
   module CoreExtensions
     # API for watching file change events
     class FileWatcher < Extension
-      attr_reader :api
+      # All defined sources.
+      Contract None => IsA['Middleman::Sources']
+      attr_reader :sources
 
-      def initialize(app, config={}, &block)
+      # The default list of ignores.
-        super
+      IGNORES = {
-      end
-
-      # Before parsing config, load the data/ directory
-      Contract None => Any
-      def before_configuration
-        @api = API.new(app)
-        app.add_to_instance :files, &method(:api)
-        app.add_to_config_context :files, &method(:api)
-      end
-
-      Contract None => Any
-      def after_configuration
-        @api.reload_path('.')
-        @api.is_ready = true
-      end
-
-      # Core File Change API class
-      class API
-        extend Forwardable
-        include Contracts
-
-        attr_reader :app
-        attr_reader :known_paths
-        attr_accessor :is_ready
-
-        def_delegator :@app, :logger
-
-        # Initialize api and internal path cache
-        def initialize(app)
-          @app = app
-          @known_paths = Set.new
-          @is_ready = false
-
-          @watchers = {
-            source: proc { |path, _| path.match(/^#{app.config[:source]}\//) },
-            library: /^(lib|helpers)\/.*\.rb$/
-          }
-
-          @ignores = {
         emacs_files: /(^|\/)\.?#/,
         tilde_files: /~$/,
-            ds_store: /\.DS_Store\//,
+        ds_store: /\.DS_Store$/,
         git: /(^|\/)\.git(ignore|modules|\/)/
       }
 
-          @on_change_callbacks = Set.new
+      # Setup the extension.
-          @on_delete_callbacks = Set.new
+      def initialize(app, config={}, &block)
+        super
+
+        # Setup source collection.
+        @sources = ::Middleman::Sources.new(app,
+                                            disable_watcher: app.config[:watcher_disable],
+                                            force_polling: app.config[:force_polling],
+                                            latency: app.config[:watcher_latency])
+
+        # Add default ignores.
+        IGNORES.each do |key, value|
+          @sources.ignore key, :all, value
         end
 
-        # Add a proc to watch paths
+        # Watch current source.
-        Contract Symbol, Or[Regexp, Proc] => Any
+        start_watching(app.config[:source])
-        def watch(name, regex=nil, &block)
-          @watchers[name] = block_given? ? block : regex
 
-          reload_path('.') if @is_ready
+        # Expose API to app and config.
+        app.add_to_instance(:files, &method(:sources))
+        app.add_to_config_context(:files, &method(:sources))
       end
 
-        # Add a proc to ignore paths
+      # Before we config, find initial files.
-        Contract Symbol, Or[Regexp, Proc] => Any
-        def ignore(name, regex=nil, &block)
-          @ignores[name] = block_given? ? block : regex
-
-          reload_path('.') if @is_ready
-        end
-
-        CallbackDescriptor = Struct.new(:proc, :matcher)
-
-        # Add callback to be run on file change
       #
-        # @param [nil,Regexp] matcher A Regexp to match the change path against
-        # @return [Array<Proc>]
-        Contract Or[Regexp, Proc] => SetOf['Middleman::CoreExtensions::FileWatcher::API::CallbackDescriptor']
-        def changed(matcher=nil, &block)
-          @on_change_callbacks << CallbackDescriptor.new(block, matcher) if block_given?
-          @on_change_callbacks
-        end
-
-        # Add callback to be run on file deletion
-        #
-        # @param [nil,Regexp] matcher A Regexp to match the deleted path against
-        # @return [Array<Proc>]
-        Contract Or[Regexp, Proc] => SetOf['Middleman::CoreExtensions::FileWatcher::API::CallbackDescriptor']
-        def deleted(matcher=nil, &block)
-          @on_delete_callbacks << CallbackDescriptor.new(block, matcher) if block_given?
-          @on_delete_callbacks
-        end
-
-        # Notify callbacks that a file changed
-        #
-        # @param [Pathname] path The file that changed
       # @return [void]
-        Contract Or[Pathname, String] => Any
+      Contract None => Any
-        def did_change(path)
+      def before_configuration
-          path = Pathname(path)
+        @sources.find_new_files!
-          logger.debug "== File Change: #{path}"
-          @known_paths << path
-          run_callbacks(path, :changed)
       end
 
-        # Notify callbacks that a file was deleted
+      # After we config, find new files since config can change paths.
       #
-        # @param [Pathname] path The file that was deleted
       # @return [void]
-        Contract Or[Pathname, String] => Any
+      Contract None => Any
-        def did_delete(path)
+      def after_configuration
-          path = Pathname(path)
+        if @original_source_dir != app.config[:source]
-          logger.debug "== File Deletion: #{path}"
+          @watcher.update_path(app.config[:source])
-          @known_paths.delete(path)
-          run_callbacks(path, :deleted)
         end
 
-        # Manually trigger update events
+        @sources.start!
-        #
+        @sources.find_new_files!
-        # @param [Pathname] path The path to reload
-        # @param [Boolean] only_new Whether we only look for new files
-        # @return [void]
-        Contract Or[String, Pathname], Maybe[Bool] => Any
-        def reload_path(path, only_new=false)
-          # chdir into the root directory so Pathname can work with relative paths
-          Dir.chdir @app.root_path do
-            path = Pathname(path)
-            return unless path.exist?
-
-            glob = (path + '**').to_s
-            subset = @known_paths.select { |p| p.fnmatch(glob) }
-
-            ::Middleman::Util.all_files_under(path, &method(:ignored?)).each do |filepath|
-              next if only_new && subset.include?(filepath)
-
-              subset.delete(filepath)
-              did_change(filepath)
-            end
-
-            subset.each(&method(:did_delete)) unless only_new
-          end
-        end
-
-        # Like reload_path, but only triggers events on new files
-        #
-        # @param [Pathname] path The path to reload
-        # @return [void]
-        Contract Pathname => Any
-        def find_new_files(path)
-          reload_path(path, true)
-        end
-
-        Contract String => Bool
-        def exists?(path)
-          p = Pathname(path)
-          p = p.relative_path_from(Pathname(@app.root)) unless p.relative?
-          @known_paths.include?(p)
-        end
-
-        # Whether this path is ignored
-        # @param [Pathname] path
-        # @return [Boolean]
-        Contract Or[String, Pathname] => Bool
-        def ignored?(path)
-          path = path.to_s
-
-          watched = @watchers.values.any? { |validator| matches?(validator, path) }
-          not_ignored = @ignores.values.none? { |validator| matches?(validator, path) }
-
-          !(watched && not_ignored)
-        end
-
-        Contract Or[Regexp, RespondTo[:call]], String => Bool
-        def matches?(validator, path)
-          if validator.is_a? Regexp
-            !!validator.match(path)
-          else
-            !!validator.call(path, @app)
-          end
       end
 
       protected
 
-        # Notify callbacks for a file given an array of callbacks
+      # Watch the source directory.
       #
-        # @param [Pathname] path The file that was changed
-        # @param [Symbol] callbacks_name The name of the callbacks method
       # @return [void]
-        Contract Or[Pathname, String], Symbol => Any
+      Contract String => Any
-        def run_callbacks(path, callbacks_name)
+      def start_watching(dir)
-          path = path.to_s
+        @original_source_dir = dir
-          send(callbacks_name).each do |callback|
+        @watcher = @sources.watch :source, path: File.join(app.root, dir)
-            next unless callback[:matcher].nil? || path.match(callback[:matcher])
-            @app.instance_exec(path, &callback[:proc])
-          end
-        end
       end
     end
   end

middleman-core/lib/middleman-core/core_extensions/front_matter.rb

@@ -27,17 +27,16 @@ module Middleman::CoreExtensions
     end
 
     def before_configuration
-      app.files.changed(&method(:clear_data))
+      app.files.changed(:source, &method(:clear_data))
-      app.files.deleted(&method(:clear_data))
     end
 
     # @return Array<Middleman::Sitemap::Resource>
     Contract ResourceList => ResourceList
     def manipulate_resource_list(resources)
       resources.each do |resource|
-        next if resource.source_file.blank?
+        next if resource.source_file.nil?
 
-        fmdata = data(resource.source_file).first.dup
+        fmdata = data(resource.source_file[:full_path].to_s).first.dup
 
         # Copy over special options
         # TODO: Should we make people put these under "options" instead of having
@@ -61,42 +60,35 @@ module Middleman::CoreExtensions
     # Get the template data from a path
     # @param [String] path
     # @return [String]
-    Contract String => String
+    Contract String => Maybe[String]
     def template_data_for_file(path)
       data(path).last
     end
 
     Contract String => [Hash, Maybe[String]]
     def data(path)
-      p = normalize_path(path)
+      file = app.files.find(:source, path)
-      @cache[p] ||= frontmatter_and_content(p)
+
+      return [{}, nil] unless file
+
+      @cache[file[:full_path]] ||= frontmatter_and_content(file[:full_path])
     end
 
-    def clear_data(file)
+    Contract ArrayOf[IsA['Middleman::SourceFile']], ArrayOf[IsA['Middleman::SourceFile']] => Any
-      # Copied from Sitemap::Store#file_to_path, but without
+    def clear_data(updated_files, removed_files)
-      # removing the file extension
+      (updated_files + removed_files).each do |file|
-      file = File.join(app.root, file)
+        @cache.delete(file[:full_path])
-      prefix = app.source_dir.sub(/\/$/, '') + '/'
+      end
-      return unless file.include?(prefix)
-      path = file.sub(prefix, '')
-
-      @cache.delete(path)
     end
 
     # Get the frontmatter and plain content from a file
     # @param [String] path
     # @return [Array<Middleman::Util::HashWithIndifferentAccess, String>]
-    Contract String => [Hash, Maybe[String]]
+    Contract Pathname => [Hash, Maybe[String]]
-    def frontmatter_and_content(path)
+    def frontmatter_and_content(full_path)
-      full_path = if Pathname(path).relative?
-        File.join(app.source_dir, path)
-      else
-        path
-      end
-
       data = {}
 
-      return [data, nil] if !app.files.exists?(full_path) || ::Middleman::Util.binary?(full_path)
+      return [data, nil] if ::Middleman::Util.binary?(full_path)
 
       content = File.read(full_path)
 
@@ -121,7 +113,7 @@ module Middleman::CoreExtensions
     # Parse YAML frontmatter out of a string
     # @param [String] content
     # @return [Array<Hash, String>]
-    Contract String, String => Maybe[[Hash, String]]
+    Contract String, Pathname => Maybe[[Hash, String]]
     def parse_yaml_front_matter(content, full_path)
       yaml_regex = /\A(---\s*\n.*?\n?)^(---\s*$\n?)/m
       if content =~ yaml_regex
@@ -146,7 +138,7 @@ module Middleman::CoreExtensions
     # Parse JSON frontmatter out of a string
     # @param [String] content
     # @return [Array<Hash, String>]
-    Contract String, String => Maybe[[Hash, String]]
+    Contract String, Pathname => Maybe[[Hash, String]]
     def parse_json_front_matter(content, full_path)
       json_regex = /\A(;;;\s*\n.*?\n?)^(;;;\s*$\n?)/m
 
@@ -169,9 +161,5 @@ module Middleman::CoreExtensions
     rescue
       [{}, content]
     end
-
-    def normalize_path(path)
-      path.sub(%r{^#{Regexp.escape(app.source_dir)}\/}, '')
-    end
   end
 end

middleman-core/lib/middleman-core/core_extensions/i18n.rb

@@ -18,28 +18,24 @@ class Middleman::CoreExtensions::Internationalization < ::Middleman::Extension
 
     locales_file_path = options[:data]
 
-    # Tell the file watcher to observe the :locales_dir
+    # Tell the file watcher to observe the :data_dir
-    app.files.watch :locales do |path, _app|
+    app.files.watch :locales,
-      path.match(/^#{locales_file_path}\/.*(yml|yaml)$/)
+                    path: File.join(app.root, locales_file_path),
-    end
+                    ignore: proc { |f| !(/.*(rb|yml|yaml)$/.match(f[:relative_path])) }
 
-    app.files.reload_path(locales_file_path)
+    # Setup data files before anything else so they are available when
-
+    # parsing config.rb
-    @locales_glob = File.join(locales_file_path, '**', '*.{rb,yml,yaml}')
+    app.files.changed(:locales, &method(:on_file_changed))
-    @locales_regex = convert_glob_to_regex(@locales_glob)
 
     @maps = {}
     @mount_at_root = options[:mount_at_root].nil? ? langs.first : options[:mount_at_root]
 
-    configure_i18n
-
-    logger.info "== Locales: #{langs.join(', ')} (Default #{@mount_at_root})"
-
     # Don't output localizable files
     app.ignore File.join(options[:templates_dir], '**')
 
-    app.files.changed(&method(:on_file_changed))
+    configure_i18n
-    app.files.deleted(&method(:on_file_changed))
+
+    logger.info "== Locales: #{langs.join(', ')} (Default #{@mount_at_root})"
   end
 
   helpers do
@@ -87,22 +83,16 @@ class Middleman::CoreExtensions::Internationalization < ::Middleman::Extension
 
   private
 
-  def on_file_changed(file)
+  Contract Any, Any => Any
-    return unless @locales_regex =~ file
+  def on_file_changed(_updated_files, _removed_files)
-
     @_langs = nil # Clear langs cache
+
+    # TODO, add new file to ::I18n.load_path
     ::I18n.reload!
   end
 
-  Contract String => Regexp
-  def convert_glob_to_regex(glob)
-    # File.fnmatch doesn't support brackets: {rb,yml,yaml}
-    regex = glob.sub(/\./, '\.').sub(File.join('**', '*'), '.*').sub(/\//, '\/').sub('{rb,yml,yaml}', '(rb|ya?ml)')
-    %r{^#{regex}}
-  end
-
   def configure_i18n
-    ::I18n.load_path += Dir[File.join(app.root, @locales_glob)]
+    ::I18n.load_path += app.files.by_type(:locales).files.map { |p| p[:full_path].to_s }
     ::I18n.reload!
 
     ::I18n.default_locale = @mount_at_root
@@ -116,12 +106,12 @@ class Middleman::CoreExtensions::Internationalization < ::Middleman::Extension
     if options[:langs]
       Array(options[:langs]).map(&:to_sym)
     else
-      known_langs = app.files.known_paths.select do |p|
+      known_langs = app.files.by_type(:locales).files.select do |p|
-        p.to_s.match(@locales_regex) && (p.to_s.split(File::SEPARATOR).length == 2)
+        p[:relative_path].to_s.split(File::SEPARATOR).length == 1
       end
 
       known_langs.map { |p|
-        File.basename(p.to_s).sub(/\.ya?ml$/, '').sub(/\.rb$/, '')
+        File.basename(p[:relative_path].to_s).sub(/\.ya?ml$/, '').sub(/\.rb$/, '')
       }.sort.map(&:to_sym)
     end
   end

middleman-core/lib/middleman-core/core_extensions/routing.rb

@@ -13,7 +13,7 @@ module Middleman
       end
 
       def before_configuration
-        app.add_to_config_context :page, &method(:page)
+        app.add_to_config_context(:page, &method(:page))
       end
 
       # @return Array<Middleman::Sitemap::Resource>
@@ -59,7 +59,7 @@ module Middleman
         if path.is_a?(String) && !path.include?('*')
           # Normalize path
           path = Middleman::Util.normalize_path(path)
-          if path.end_with?('/') || File.directory?(File.join(@app.source_dir, path))
+          if path.end_with?('/') || app.files.by_type(:source).watchers.any? { |w| (w.directory + Pathname(path)).directory? }
             path = File.join(path, @app.config[:index_file])
           end
         end

middleman-core/lib/middleman-core/core_extensions/show_exceptions.rb

@@ -6,6 +6,8 @@ module Middleman::CoreExtensions
     def initialize(app, options_hash={}, &block)
       super
 
+      return if app.config.defines_setting? :show_exceptions
+
       app.config.define_setting :show_exceptions, !!ENV['TEST'], 'Whether to catch and display exceptions'
     end
 

middleman-core/lib/middleman-core/extensions.rb

@@ -10,6 +10,7 @@ module Middleman
     @auto_activate = {
       # Activate before the Sitemap is instantiated
       before_sitemap: Set.new,
+
       # Activate the extension before `config.rb` and the `:before_configuration` hook.
       before_configuration: Set.new
     }

middleman-core/lib/middleman-core/extensions/automatic_alt_tags.rb

@@ -12,13 +12,14 @@ class Middleman::Extensions::AutomaticAltTags < ::Middleman::Extension
       unless path.include?('://')
         params[:alt] ||= ''
 
-        real_path = path
+        real_path = path.dup
         real_path = File.join(images_dir, real_path) unless real_path.start_with?('/')
-        full_path = File.join(source_dir, real_path)
 
-        if File.exist?(full_path)
+        file = app.files.find(:source, real_path)
+
+        if file && file[:full_path].exist?
           begin
-            alt_text = File.basename(full_path, '.*')
+            alt_text = File.basename(file[:full_path].to_s, '.*')
             alt_text.capitalize!
             params[:alt] = alt_text
           end

middleman-core/lib/middleman-core/extensions/automatic_image_sizes.rb

@@ -18,13 +18,14 @@ class Middleman::Extensions::AutomaticImageSizes < ::Middleman::Extension
       if !params.key?(:width) && !params.key?(:height) && !path.include?('://')
         params[:alt] ||= ''
 
-        real_path = path
+        real_path = path.dup
         real_path = File.join(config[:images_dir], real_path) unless real_path.start_with?('/')
-        full_path = File.join(source_dir, real_path)
 
-        if File.exist?(full_path)
+        file = app.files.find(:source, real_path)
+
+        if file && file[:full_path].exist?
           begin
-            width, height = ::FastImage.size(full_path, raise_on_failure: true)
+            width, height = ::FastImage.size(file[:full_path].to_s, raise_on_failure: true)
             params[:width]  = width
             params[:height] = height
           rescue FastImage::UnknownImageType

middleman-core/lib/middleman-core/file_renderer.rb

@@ -100,12 +100,13 @@ module Middleman
     # Get the template data from a path
     # @param [String] path
     # @return [String]
-    Contract String => String
+    Contract None => String
     def template_data_for_file
       if @app.extensions[:front_matter]
-        @app.extensions[:front_matter].template_data_for_file(@path)
+        @app.extensions[:front_matter].template_data_for_file(@path) || ''
       else
-        File.read(File.expand_path(@path, source_dir))
+        file = @app.files.find(:source, @path)
+        file.read if file
       end
     end
 

middleman-core/lib/middleman-core/meta_pages/sitemap_resource.rb

@@ -39,7 +39,7 @@ module Middleman
         build_path = 'Not built' if ignored?
         props['Build Path'] = build_path if @resource.path != build_path
         props['URL'] = content_tag(:a, @resource.url, href: @resource.url) unless ignored?
-        props['Source File'] = @resource.source_file.sub(/^#{Regexp.escape(ENV['MM_ROOT'] + '/')}/, '')
+        props['Source File'] = @resource.source_file[:full_path].to_s
 
         data = @resource.data
         props['Data'] = data.inspect unless data.empty?

middleman-core/lib/middleman-core/preview_server.rb

@@ -58,10 +58,6 @@ module Middleman
           # if the user closed their terminal STDOUT/STDERR won't exist
         end
 
-        if @listener
-          @listener.stop
-          @listener = nil
-        end
         unmount_instance
       end
 
@@ -103,6 +99,30 @@ module Middleman
 
         app = ::Middleman::Application.new do
           config[:environment] = opts[:environment].to_sym if opts[:environment]
+          config[:watcher_disable] = opts[:disable_watcher]
+          config[:watcher_force_polling] = opts[:force_polling]
+          config[:watcher_latency] = opts[:latency]
+
+          ready do
+            match_against = [
+              %r{^config\.rb$},
+              %r{^environments/[^\.](.*)\.rb$},
+              %r{^lib/[^\.](.*)\.rb$},
+              %r{^#{@app.config[:helpers_dir]}/[^\.](.*)\.rb$}
+            ]
+
+            # config.rb
+            files.watch :reload,
+                        path: root,
+                        ignored: proc { |file|
+                          match_against.none? { |m| file[:relative_path].to_s.match(m) }
+                        }
+          end
+        end
+
+        app.files.changed :reload do
+          $mm_reload = true
+          @webrick.stop
         end
 
         # Add in the meta pages application
@@ -114,41 +134,6 @@ module Middleman
         app
       end
 
-      def start_file_watcher
-        return if @listener || @options[:disable_watcher]
-
-        # Watcher Library
-        require 'listen'
-
-        options = { force_polling: @options[:force_polling] }
-        options[:latency] = @options[:latency] if @options[:latency]
-
-        @listener = Listen.to(Dir.pwd, options) do |modified, added, removed|
-          added_and_modified = (modified + added)
-
-          # See if the changed file is config.rb or lib/*.rb
-          if needs_to_reload?(added_and_modified + removed)
-            $mm_reload = true
-            @webrick.stop
-          else
-            added_and_modified.each do |path|
-              relative_path = Pathname(path).relative_path_from(Pathname(Dir.pwd)).to_s
-              next if app.files.ignored?(relative_path)
-              app.files.did_change(relative_path)
-            end
-
-            removed.each do |path|
-              relative_path = Pathname(path).relative_path_from(Pathname(Dir.pwd)).to_s
-              next if app.files.ignored?(relative_path)
-              app.files.did_delete(relative_path)
-            end
-          end
-        end
-
-        # Don't block this thread
-        @listener.start
-      end
-
       # Trap some interupt signals and shut down smoothly
       # @return [void]
       def register_signal_handlers
@@ -196,8 +181,6 @@ module Middleman
 
         @webrick ||= setup_webrick(@options[:debug] || false)
 
-        start_file_watcher
-
         rack_app = ::Middleman::Rack.new(@app).to_app
         @webrick.mount '/', ::Rack::Handler::WEBrick, rack_app
       end
@@ -206,33 +189,12 @@ module Middleman
       # @return [void]
       def unmount_instance
         @webrick.unmount '/'
+
+        @app.shutdown!
+
         @app = nil
       end
 
-      # Whether the passed files are config.rb, lib/*.rb or helpers
-      # @param [Array<String>] paths Array of paths to check
-      # @return [Boolean] Whether the server needs to reload
-      def needs_to_reload?(paths)
-        match_against = [
-          %r{^/config\.rb},
-          %r{^/environments/[^\.](.*)\.rb$},
-          %r{^/lib/[^\.](.*)\.rb$},
-          %r{^/#{@app.config[:helpers_dir]}/[^\.](.*)\.rb$}
-        ]
-
-        if @options[:reload_paths]
-          @options[:reload_paths].split(',').each do |part|
-            match_against << %r{^#{part}}
-          end
-        end
-
-        paths.any? do |path|
-          match_against.any? do |matcher|
-            path.sub(@app.root, '').match matcher
-          end
-        end
-      end
-
       # Returns the URI the preview server will run on
       # @return [URI]
       def uri

middleman-core/lib/middleman-core/rack.rb

@@ -123,7 +123,7 @@ module Middleman
     # Immediately send static file
     def send_file(resource, env)
       file      = ::Rack::File.new nil
-      file.path = resource.source_file
+      file.path = resource.source_file[:full_path]
       response = file.serving(env)
       status = response[0]
       response[1]['Content-Encoding'] = 'gzip' if %w(.svgz .gz).include?(resource.ext)

middleman-core/lib/middleman-core/renderers/less.rb

@@ -10,15 +10,17 @@ module Middleman
         # Default less options
         app.config.define_setting :less, {}, 'LESS compiler options'
 
-        app.after_configuration do
-          ::Less.paths << File.join(source_dir, config[:css_dir])
-        end
-
         # Tell Tilt to use it as well (for inline sass blocks)
         ::Tilt.register 'less', LocalLoadingLessTemplate
         ::Tilt.prefer(LocalLoadingLessTemplate)
       end
 
+      def after_configuration
+        app.files.by_type(:source).watchers.each do |source|
+          ::Less.paths << (source.directory + app.config[:css_dir]).to_s
+        end
+      end
+
       # A SassTemplate for Tilt which outputs debug messages
       class LocalLoadingLessTemplate < ::Tilt::LessTemplate
         def prepare

middleman-core/lib/middleman-core/renderers/liquid.rb

@@ -7,7 +7,14 @@ module Middleman
     class Liquid < Middleman::Extension
       # After config, setup liquid partial paths
       def after_configuration
-        ::Liquid::Template.file_system = ::Liquid::LocalFileSystem.new(app.source_dir)
+        ::Liquid::Template.file_system = self
+      end
+
+      # Called by Liquid to retrieve a template file
+      def read_template_file(template_path, _)
+        file = app.files.find(:source, "_#{template_path}.liquid")
+        raise ::Liquid::FileSystemError, "No such template '#{template_path}'" unless file
+        File.read(file[:full_path])
       end
 
       # @return Array<Middleman::Sitemap::Resource>
@@ -16,7 +23,8 @@ module Middleman
         return resources unless app.extensions[:data]
 
         resources.each do |resource|
-          next unless resource.source_file =~ %r{\.liquid$}
+          next if resource.source_file.nil?
+          next unless resource.source_file[:relative_path].to_s =~ %r{\.liquid$}
 
           # Convert data object into a hash for liquid
           resource.add_metadata locals: { data: app.extensions[:data].data_store.to_h }

middleman-core/lib/middleman-core/renderers/sass.rb

@@ -38,6 +38,8 @@ module Middleman
       def initialize(app, options={}, &block)
         super
 
+        app.files.ignore :sass_cache, :source, /(^|\/)\.sass-cache\//
+
         opts = { output_style: :nested }
         opts[:line_comments] = false if ENV['TEST']
 
@@ -59,10 +61,6 @@ module Middleman
         require 'middleman-core/renderers/sass_functions'
       end
 
-      def before_configuration
-        app.files.watch :sass_cache, /(^|\/)\.sass-cache\//
-      end
-
       # A SassTemplate for Tilt which outputs debug messages
       class SassPlusCSSFilenameTemplate < ::Tilt::SassTemplate
         def initialize(*args, &block)
@@ -111,11 +109,7 @@ module Middleman
           }
 
           if ctx.is_a?(::Middleman::TemplateContext) && file
-            location_of_sass_file = ctx.source_dir
+            more_opts[:css_filename] = file.sub(/\.s[ac]ss$/, '')
-
-            parts = basename.split('.')
-            parts.pop
-            more_opts[:css_filename] = File.join(location_of_sass_file, ctx.config[:css_dir], parts.join('.'))
           end
 
           options.merge(more_opts)

middleman-core/lib/middleman-core/sitemap/extensions/ignores.rb

@@ -6,13 +6,13 @@ module Middleman
         def initialize(app, config={}, &block)
           super
 
-          @app.add_to_config_context :ignore, &method(:create_ignore)
+          @app.add_to_config_context(:ignore, &method(:create_ignore))
-          @app.define_singleton_method :ignore, &method(:create_ignore)
+          @app.define_singleton_method(:ignore, &method(:create_ignore))
 
           # Array of callbacks which can ass ignored
           @ignored_callbacks = Set.new
 
-          @app.sitemap.define_singleton_method :ignored?, &method(:ignored?)
+          @app.sitemap.define_singleton_method(:ignored?, &method(:ignored?))
         end
 
         # Ignore a path or add an ignore callback

middleman-core/lib/middleman-core/sitemap/extensions/on_disk.rb

@@ -24,32 +24,21 @@ module Middleman
 
         Contract None => Any
         def before_configuration
-          app.files.changed(&method(:touch_file))
+          app.files.changed(:source, &method(:update_files))
-          app.files.deleted(&method(:remove_file))
+        end
+
+        def ignored?(file)
+          @app.config[:ignored_sitemap_matchers].any? do |_, callback|
+            callback.call(file, @app)
+          end
         end
 
         # Update or add an on-disk file path
         # @param [String] file
         # @return [void]
-        Contract String => Any
+        Contract ArrayOf[IsA['Middleman::SourceFile']], ArrayOf[IsA['Middleman::SourceFile']] => Any
-        def touch_file(file)
+        def update_files(updated_files, removed_files)
-          return false if File.directory?(file)
+          return if (updated_files + removed_files).all?(&method(:ignored?))
-
-          begin
-            @app.sitemap.file_to_path(file)
-          rescue
-            return
-          end
-
-          ignored = @app.config[:ignored_sitemap_matchers].any? do |_, callback|
-            if callback.arity == 1
-              callback.call(file)
-            else
-              callback.call(file, @app)
-            end
-          end
-
-          @file_paths_on_disk << file unless ignored
 
           # Rebuild the sitemap any time a file is touched
           # in case one of the other manipulators
@@ -62,29 +51,19 @@ module Middleman
           @app.sitemap.ensure_resource_list_updated! unless waiting_for_ready || @app.build?
         end
 
-        # Remove a file from the store
+        def files_for_sitemap
-        # @param [String] file
+          @app.files.by_type(:source).files.reject(&method(:ignored?))
-        # @return [void]
-        Contract String => Any
-        def remove_file(file)
-          return unless @file_paths_on_disk.delete?(file)
-
-          @app.sitemap.rebuild_resource_list!(:removed_file)
-
-          # Force sitemap rebuild so the next request is ready to go.
-          # Skip this during build because the builder will control sitemap refresh.
-          @app.sitemap.ensure_resource_list_updated! unless waiting_for_ready || @app.build?
         end
 
         # Update the main sitemap resource list
         # @return Array<Middleman::Sitemap::Resource>
         Contract ResourceList => ResourceList
         def manipulate_resource_list(resources)
-          resources + @file_paths_on_disk.map do |file|
+          resources + files_for_sitemap.map do |file|
             ::Middleman::Sitemap::Resource.new(
               @app.sitemap,
               @app.sitemap.file_to_path(file),
-              File.join(@app.root, file)
+              file
             )
           end
         end

middleman-core/lib/middleman-core/sitemap/extensions/proxies.rb

@@ -9,7 +9,7 @@ module Middleman
         def initialize(app, config={}, &block)
           super
 
-          @app.add_to_config_context :proxy, &method(:create_proxy)
+          @app.add_to_config_context(:proxy, &method(:create_proxy))
           @app.define_singleton_method(:proxy, &method(:create_proxy))
 
           @proxy_configs = Set.new
@@ -125,7 +125,7 @@ module Middleman
         resource
       end
 
-      Contract None => String
+      Contract None => IsA['Middleman::SourceFile']
       def source_file
         target_resource.source_file
       end

middleman-core/lib/middleman-core/sitemap/extensions/redirects.rb

@@ -10,7 +10,7 @@ module Middleman
         def initialize(app, config={}, &block)
           super
 
-          @app.add_to_config_context :redirect, &method(:create_redirect)
+          @app.add_to_config_context(:redirect, &method(:create_redirect))
 
           @redirects = {}
         end

middleman-core/lib/middleman-core/sitemap/extensions/request_endpoints.rb

@@ -9,7 +9,7 @@ module Middleman
         def initialize(app, config={}, &block)
           super
 
-          @app.add_to_config_context :endpoint, &method(:create_endpoint)
+          @app.add_to_config_context(:endpoint, &method(:create_endpoint))
 
           @endpoints = {}
         end

middleman-core/lib/middleman-core/sitemap/extensions/traversal.rb

@@ -76,8 +76,9 @@ module Middleman
             return true
           end
 
-          full_path = File.join(@app.source_dir, eponymous_directory_path)
+          @app.files.by_type(:source).watchers.any? do |source|
-          File.exist?(full_path) && File.directory?(full_path)
+            (source.directory + Pathname(eponymous_directory_path)).directory?
+          end
         end
 
         # The path for this resource if it were a directory, and not a file

middleman-core/lib/middleman-core/sitemap/resource.rb

@@ -23,6 +23,7 @@ module Middleman
 
       # The on-disk source file for this resource, if there is one
       # @return [String]
+      Contract None => Maybe[IsA['Middleman::SourceFile']]
       attr_reader :source_file
 
       # The path to use when requesting this resource. Normally it's
@@ -41,6 +42,7 @@ module Middleman
       # @param [Middleman::Sitemap::Store] store
       # @param [String] path
       # @param [String] source_file
+      Contract IsA['Middleman::Sitemap::Store'], String, Maybe[IsA['Middleman::SourceFile']] => Any
       def initialize(store, path, source_file=nil)
         @store       = store
         @app         = @store.app
@@ -60,7 +62,7 @@ module Middleman
       Contract None => Bool
       def template?
         return false if source_file.nil?
-        !::Tilt[source_file].nil?
+        !::Tilt[source_file[:full_path].to_s].nil?
       end
 
       # Merge in new metadata specific to this resource.
@@ -108,11 +110,9 @@ module Middleman
       # @return [String]
       Contract Hash, Hash => String
       def render(opts={}, locs={})
-        return ::Middleman::FileRenderer.new(@app, source_file).template_data_for_file unless template?
+        return ::Middleman::FileRenderer.new(@app, source_file[:full_path].to_s).template_data_for_file unless template?
 
-        relative_source = Pathname(source_file).relative_path_from(Pathname(@app.root))
+        ::Middleman::Util.instrument 'render.resource', path: source_file[:full_path].to_s, destination_path: destination_path do
-
-        ::Middleman::Util.instrument 'render.resource', path: relative_source, destination_path: destination_path do
           md   = metadata
           opts = md[:options].deep_merge(opts)
           locs = md[:locals].deep_merge(locs)
@@ -123,7 +123,7 @@ module Middleman
             opts[:layout] = false if %w(.js .json .css .txt).include?(ext)
           end
 
-          renderer = ::Middleman::TemplateRenderer.new(@app, source_file)
+          renderer = ::Middleman::TemplateRenderer.new(@app, source_file[:full_path].to_s)
           renderer.render(locs, opts)
         end
       end
@@ -146,7 +146,7 @@ module Middleman
       # @return [Boolean]
       Contract None => Bool
       def binary?
-        !source_file.nil? && ::Middleman::Util.binary?(source_file)
+        !source_file.nil? && ::Middleman::Util.binary?(source_file[:full_path].to_s)
       end
 
       # Ignore a resource directly, without going through the whole
@@ -165,7 +165,7 @@ module Middleman
         # Ignore based on the source path (without template extensions)
         return true if @app.sitemap.ignored?(path)
         # This allows files to be ignored by their source file name (with template extensions)
-        !self.is_a?(ProxyResource) && @app.sitemap.ignored?(source_file.sub("#{@app.source_dir}/", ''))
+        !self.is_a?(ProxyResource) && @app.sitemap.ignored?(source_file[:relative_path].to_s)
       end
 
       # The preferred MIME content type for this resource based on extension or metadata

middleman-core/lib/middleman-core/sitemap/store.rb

@@ -142,21 +142,16 @@ module Middleman
       # Get the URL path for an on-disk file
       # @param [String] file
       # @return [String]
-      Contract String => String
+      Contract IsA['Middleman::SourceFile'] => String
       def file_to_path(file)
-        file = File.expand_path(file, @app.root)
+        relative_path = file[:relative_path].to_s
-
-        prefix = @app.source_dir.sub(/\/$/, '') + '/'
-        raise "'#{file}' not inside project folder '#{prefix}" unless file.start_with?(prefix)
-
-        path = file.sub(prefix, '')
 
         # Replace a file name containing automatic_directory_matcher with a folder
         unless @app.config[:automatic_directory_matcher].nil?
-          path = path.gsub(@app.config[:automatic_directory_matcher], '/')
+          relative_path = relative_path.gsub(@app.config[:automatic_directory_matcher], '/')
         end
 
-        extensionless_path(path)
+        extensionless_path(relative_path)
       end
 
       # Get a path without templating extensions

middleman-core/lib/middleman-core/sources.rb

@@ -0,0 +1,310 @@
+require 'middleman-core/contracts'
+
+module Middleman
+  # The standard "record" that contains information about a file on disk.
+  SourceFile = Struct.new :relative_path, :full_path, :directory, :type
+
+  # Sources handle multiple on-disk collections of files which make up
+  # a Middleman project. They are separated by `type` which can then be
+  # queried. For example, the `source` type represents all content that
+  # the sitemap uses to build a project. The `data` type represents YAML
+  # data. The `locales` type represents localization YAML, and so on.
+  class Sources
+    extend Forwardable
+    include Contracts
+
+    # A reference to the current app.
+    Contract None => IsA['Middleman::Application']
+    attr_reader :app
+
+    # Duck-typed definition of a valid source watcher
+    HANDLER = RespondTo[:changed]
+
+    # Config
+    Contract None => Hash
+    attr_reader :options
+
+    # Reference to the global logger.
+    def_delegator :@app, :logger
+
+    # Built-in types
+    # :source, :data, :locales, :reload
+
+    # Create a new collection of sources.
+    #
+    # @param [Middleman::Application] app The parent app.
+    # @param [Hash] options Global options.
+    # @param [Array] watchers Default watchers.
+    Contract IsA['Middleman::Application'], Maybe[Hash], Maybe[Array] => Any
+    def initialize(app, options={}, watchers=[])
+      @app = app
+      @watchers = watchers
+      @sorted_watchers = @watchers.dup.freeze
+
+      @options = options
+
+      # Set of procs wanting to be notified of changes
+      @on_change_callbacks = []
+
+      # Global ignores
+      @ignores = {}
+
+      # Whether we're "running", which means we're in a stable
+      # watch state after all initialization and config.
+      @running = false
+
+      @update_count = 0
+      @last_update_count = -1
+
+      # When the app is about to shut down, stop our watchers.
+      @app.before_shutdown(&method(:stop!))
+    end
+
+    # Add a proc to ignore paths with either a regex or block.
+    #
+    # @param [Symbol] name A name for the ignore.
+    # @param [Symbol] type The type of content to apply the ignore to.
+    # @param [Regexp] regex Ignore by path regex.
+    # @param [Proc] block Ignore by block evaluation.
+    # @return [void]
+    Contract Symbol, Symbol, Or[Regexp, Proc] => Any
+    def ignore(name, type, regex=nil, &block)
+      @ignores[name] = { type: type, validator: (block_given? ? block : regex) }
+
+      bump_count
+      find_new_files! if @running
+    end
+
+    # Whether this path is ignored.
+    #
+    # @param [Middleman::SourceFile] file The file to check.
+    # @return [Boolean]
+    Contract SourceFile => Bool
+    def globally_ignored?(file)
+      @ignores.values.any? do |descriptor|
+        ((descriptor[:type] == :all) || (file[:type] == descriptor[:type])) &&
+        matches?(descriptor[:validator], file)
+      end
+    end
+
+    # Connect a new watcher. Can either be a type with options, which will
+    # create a `SourceWatcher` or you can pass in an instantiated class which
+    # responds to #changed and #deleted
+    #
+    # @param [Symbol, #changed, #deleted] type_or_handler The handler.
+    # @param [Hash] options The watcher options.
+    # @return [#changed, #deleted]
+    Contract Or[Symbol, HANDLER], Maybe[Hash] => HANDLER
+    def watch(type_or_handler, options={})
+      handler = if type_or_handler.is_a? Symbol
+        SourceWatcher.new(self, type_or_handler, options.delete(:path), options)
+      else
+        type_or_handler
+      end
+
+      @watchers << handler
+
+      # The index trick is used so that the sort is stable - watchers with the same priority
+      # will always be ordered in the same order as they were registered.
+      n = 0
+      @sorted_watchers = @watchers.sort_by do |w|
+        priority = w.options.fetch(:priority, 50)
+        n += 1
+        [priority, n]
+      end.reverse.freeze
+
+      handler.changed(&method(:did_change))
+
+      if @running
+        handler.poll_once!
+        handler.listen!
+      end
+
+      handler
+    end
+
+    # A list of registered watchers
+    Contract None => ArrayOf[HANDLER]
+    def watchers
+      @sorted_watchers
+    end
+
+    # Disconnect a specific watcher.
+    #
+    # @param [SourceWatcher] watcher The watcher to remove.
+    # @return [void]
+    Contract RespondTo[:changed] => Any
+    def unwatch(watcher)
+      @watchers.delete(watcher)
+
+      watcher.unwatch
+
+      bump_count
+    end
+
+    # Filter the collection of watchers by a type.
+    #
+    # @param [Symbol] type The watcher type.
+    # @return [Middleman::Sources]
+    Contract Symbol => ::Middleman::Sources
+    def by_type(type)
+      self.class.new @app, @options, watchers.select { |d| d.type == type }
+    end
+
+    # Get all files for this collection of watchers.
+    #
+    # @return [Array<Middleman::SourceFile>]
+    Contract None => ArrayOf[SourceFile]
+    def files
+      watchers.map(&:files).flatten.uniq { |f| f[:relative_path] }
+    end
+
+    # Find a file given a type and path.
+    #
+    # @param [Symbol] type The file "type".
+    # @param [String] path The file path.
+    # @param [Boolean] glob If the path contains wildcard or glob characters.
+    # @return [Middleman::SourceFile, nil]
+    Contract Symbol, String, Maybe[Bool] => Maybe[SourceFile]
+    def find(type, path, glob=false)
+      watchers
+          .select { |d| d.type == type }
+          .map { |d| d.find(path, glob) }
+          .compact
+          .first
+    end
+
+    # Check if a file for a given type exists.
+    #
+    # @param [Symbol] type The file "type".
+    # @param [String] path The file path relative to it's source root.
+    # @return [Boolean]
+    Contract Symbol, String => Bool
+    def exists?(type, path)
+      watchers
+          .select { |d| d.type == type }
+          .any? { |d| d.exists?(path) }
+    end
+
+    # Check if a file for a given type exists.
+    #
+    # @param [Symbol] type The file "type".
+    # @param [String] path The file path relative to it's source root.
+    # @return [Boolean]
+    Contract Symbol, String => Maybe[HANDLER]
+    def watcher_for_path(type, path)
+      watchers
+          .select { |d| d.type == type }
+          .find { |d| d.exists?(path) }
+    end
+
+    # Manually poll all watchers for new content.
+    #
+    # @return [void]
+    Contract None => Any
+    def find_new_files!
+      return unless @update_count != @last_update_count
+
+      @last_update_count = @update_count
+      watchers.each(&:poll_once!)
+    end
+
+    # Start up all listeners.
+    #
+    # @return [void]
+    Contract None => Any
+    def start!
+      watchers.each(&:listen!)
+      @running = true
+    end
+
+    # Stop the watchers.
+    #
+    # @return [void]
+    Contract None => Any
+    def stop!
+      watchers.each(&:stop_listener!)
+      @running = false
+    end
+
+    # A callback requires a type and the proc to execute.
+    CallbackDescriptor = Struct.new :type, :proc
+
+    # Add callback to be run on file change
+    #
+    # @param [nil,Regexp] matcher A Regexp to match the change path against
+    # @return [Set<CallbackDescriptor>]
+    Contract Symbol, Proc => ArrayOf[CallbackDescriptor]
+    def changed(type, &block)
+      @on_change_callbacks << CallbackDescriptor.new(type, block)
+      @on_change_callbacks
+    end
+
+    protected
+
+    # Whether a validator matches a file.
+    #
+    # @param [Regexp, #call] validator The match validator.
+    # @param [Middleman::SourceFile] file The file to check.
+    # @return [Boolean]
+    Contract Or[Regexp, RespondTo[:call]], SourceFile => Bool
+    def matches?(validator, file)
+      path = file[:relative_path]
+      if validator.is_a? Regexp
+        !!validator.match(path.to_s)
+      else
+        !!validator.call(path, @app)
+      end
+    end
+
+    # Increment the internal counter for changes.
+    #
+    # @return [void]
+    Contract None => Any
+    def bump_count
+      @update_count += 1
+    end
+
+    # Notify callbacks that a file changed
+    #
+    # @param [Middleman::SourceFile] file The file that changed
+    # @return [void]
+    Contract ArrayOf[SourceFile], ArrayOf[SourceFile], HANDLER => Any
+    def did_change(updated_files, removed_files, watcher)
+      valid_updated = updated_files.select do |file|
+        watcher_for_path(file[:type], file[:relative_path].to_s) == watcher
+      end
+
+      valid_removed = removed_files.select do |file|
+        watcher_for_path(file[:type], file[:relative_path].to_s).nil?
+      end
+
+      return if valid_updated.empty? && valid_removed.empty?
+
+      bump_count
+      run_callbacks(@on_change_callbacks, valid_updated, valid_removed)
+    end
+
+    # Notify callbacks for a file given a set of callbacks
+    #
+    # @param [Set] callback_descriptors The registered callbacks.
+    # @param [Array<Middleman::SourceFile>] files The files that were changed.
+    # @return [void]
+    Contract ArrayOf[CallbackDescriptor], ArrayOf[SourceFile], ArrayOf[SourceFile] => Any
+    def run_callbacks(callback_descriptors, updated_files, removed_files)
+      callback_descriptors.each do |callback|
+        if callback[:type] != :all
+          callback[:proc].call(updated_files, removed_files)
+        else
+          valid_updated = updated_files.select { |f| callback[:type] == f[:type] }
+          valid_removed = removed_files.select { |f| callback[:type] == f[:type] }
+
+          callback[:proc].call(valid_updated, valid_removed)
+        end
+      end
+    end
+  end
+end
+
+# And, require the actual default implementation for a watcher.
+require 'middleman-core/sources/source_watcher.rb'

middleman-core/lib/middleman-core/sources/source_watcher.rb

@@ -0,0 +1,268 @@
+# Watcher Library
+require 'listen'
+
+require 'middleman-core/contracts'
+
+module Middleman
+  # The default source watcher implementation. Watches a directory on disk
+  # and responds to events on changes.
+  class SourceWatcher
+    extend Forwardable
+    include Contracts
+
+    # References to parent `Sources` app and `globally_ignored?` check.
+    def_delegators :@parent, :app, :globally_ignored?
+
+    # Reference to the singleton logger
+    def_delegator :app, :logger
+
+    # The type this watcher is representing
+    Contract None => Symbol
+    attr_reader :type
+
+    # The directory that is being watched
+    Contract None => Pathname
+    attr_reader :directory
+
+    # Options for configuring the watcher
+    Contract None => Hash
+    attr_reader :options
+
+    # Construct a new SourceWatcher
+    #
+    # @param [Middleman::Sources] parent The parent collection.
+    # @param [Symbol] type The watcher type.
+    # @param [String] directory The on-disk path to watch.
+    # @param [Hash] options Configuration options.
+    Contract IsA['Middleman::Sources'], Symbol, String, Hash => Any
+    def initialize(parent, type, directory, options={})
+      @parent = parent
+      @options = options
+
+      @type = type
+      @directory = Pathname(directory)
+
+      @files = {}
+
+      @validator = options.fetch(:validator, proc { true })
+      @ignored = options.fetch(:ignored, proc { false })
+
+      @disable_watcher = app.build? || @parent.options.fetch(:disable_watcher, false)
+      @force_polling = @parent.options.fetch(:force_polling, false)
+      @latency = @parent.options.fetch(:latency, nil)
+
+      @listener = nil
+
+      @on_change_callbacks = Set.new
+
+      @waiting_for_existence = !@directory.exist?
+    end
+
+    # Change the path of the watcher (if config values upstream change).
+    #
+    # @param [String] directory The new path.
+    # @return [void]
+    Contract String => Any
+    def update_path(directory)
+      @directory = Pathname(directory)
+
+      stop_listener! if @listener
+
+      update([], @files.values)
+
+      poll_once!
+
+      listen! unless @disable_watcher
+    end
+
+    # Stop watching.
+    #
+    # @return [void]
+    Contract None => Any
+    def unwatch
+      stop_listener!
+    end
+
+    # All the known files in this watcher.
+    #
+    # @return [Array<Middleman::SourceFile>]
+    Contract None => ArrayOf[IsA['Middleman::SourceFile']]
+    def files
+      @files.values
+    end
+
+    # Find a specific file in this watcher.
+    #
+    # @param [String, Pathname] path The search path.
+    # @param [Boolean] glob If the path contains wildcard characters.
+    # @return [Middleman::SourceFile, nil]
+    Contract Or[String, Pathname], Maybe[Bool] => Maybe[IsA['Middleman::SourceFile']]
+    def find(path, glob=false)
+      p = Pathname(path)
+
+      return nil if p.absolute? && !p.to_s.start_with?(@directory.to_s)
+
+      p = @directory + p if p.relative?
+
+      if glob
+        found = @files.find { |_, v| v[:relative_path].fnmatch(path) }
+        found ? found.last : nil
+      else
+        @files[p]
+      end
+    end
+
+    # Check if a file simply exists in this watcher.
+    #
+    # @param [String, Pathname] path The search path.
+    # @return [Boolean]
+    Contract Or[String, Pathname] => Bool
+    def exists?(path)
+      !find(path).nil?
+    end
+
+    # Start the `listen` gem Listener.
+    #
+    # @return [void]
+    Contract None => Any
+    def listen!
+      return if @disable_watcher || @listener || @waiting_for_existence
+
+      config = { force_polling: @force_polling }
+      config[:latency] = @latency if @latency
+
+      @listener = ::Listen.to(@directory.to_s, config, &method(:on_listener_change))
+      @listener.start
+    end
+
+    # Stop the listener.
+    #
+    # @return [void]
+    Contract None => Any
+    def stop_listener!
+      return unless @listener
+
+      @listener.stop
+      @listener = nil
+    end
+
+    # Manually trigger update events.
+    #
+    # @return [void]
+    Contract None => Any
+    def poll_once!
+      removed = @files.keys
+
+      updated = []
+
+      ::Middleman::Util.all_files_under(@directory.to_s).each do |filepath|
+        removed.delete(filepath)
+        updated << filepath
+      end
+
+      update(updated, removed)
+
+      return unless @waiting_for_existence && @directory.exist?
+
+      @waiting_for_existence = false
+      listen!
+    end
+
+    # Add callback to be run on file change
+    #
+    # @param [Proc] matcher A Regexp to match the change path against
+    # @return [Set<Proc>]
+    Contract Proc => SetOf[Proc]
+    def changed(&block)
+      @on_change_callbacks << block
+      @on_change_callbacks
+    end
+
+    # Work around this bug: http://bugs.ruby-lang.org/issues/4521
+    # where Ruby will call to_s/inspect while printing exception
+    # messages, which can take a long time (minutes at full CPU)
+    # if the object is huge or has cyclic references, like this.
+    def to_s
+      "#<Middleman::SourceWatcher:0x#{object_id} type=#{@type.inspect} directory=#{@directory.inspect}>"
+    end
+    alias_method :inspect, :to_s # Ruby 2.0 calls inspect for NoMethodError instead of to_s
+
+    protected
+
+    # The `listen` gem callback.
+    #
+    # @param [Array] modified List of modified files.
+    # @param [Array] added List of added files.
+    # @param [Array] removed List of removed files.
+    # @return [void]
+    Contract Array, Array, Array => Any
+    def on_listener_change(modified, added, removed)
+      updated = (modified + added)
+
+      return if updated.empty? && removed.empty?
+
+      update(updated.map { |s| Pathname(s) }, removed.map { |s| Pathname(s) })
+    end
+
+    # Update our internal list of files on a change.
+    #
+    # @param [String, Pathname] path The updated file path.
+    # @return [void]
+    Contract ArrayOf[Pathname], ArrayOf[Pathname] => Any
+    def update(updated_paths, removed_paths)
+      valid_updates = updated_paths
+          .map(&method(:path_to_source_file))
+          .select(&method(:valid?))
+
+      valid_updates.each do |f|
+        @files[f[:full_path]] = f
+        logger.debug "== Change (#{f[:type]}): #{f[:relative_path]}"
+      end
+
+      valid_removes = removed_paths
+          .select(&@files.method(:key?))
+          .map(&@files.method(:[]))
+          .select(&method(:valid?))
+
+      valid_removes.each do |f|
+        @files.delete(f[:full_path])
+        logger.debug "== Deletion (#{f[:type]}): #{f[:relative_path]}"
+      end
+
+      run_callbacks(@on_change_callbacks, valid_updates, valid_removes) unless valid_updates.empty? && valid_removes.empty?
+    end
+
+    # Check if this watcher should care about a file.
+    #
+    # @param [Middleman::SourceFile] file The file.
+    # @return [Boolean]
+    Contract IsA['Middleman::SourceFile'] => Bool
+    def valid?(file)
+      @validator.call(file) &&
+      !globally_ignored?(file) &&
+      !@ignored.call(file)
+    end
+
+    # Convert a path to a file resprentation.
+    #
+    # @param [Pathname] path The path.
+    # @return [Middleman::SourceFile]
+    Contract Pathname => IsA['Middleman::SourceFile']
+    def path_to_source_file(path)
+      ::Middleman::SourceFile.new(
+        path.relative_path_from(@directory), path, @directory, @type)
+    end
+
+    # Notify callbacks for a file given an array of callbacks
+    #
+    # @param [Pathname] path The file that was changed
+    # @param [Symbol] callbacks_name The name of the callbacks method
+    # @return [void]
+    Contract Set, ArrayOf[IsA['Middleman::SourceFile']], ArrayOf[IsA['Middleman::SourceFile']] => Any
+    def run_callbacks(callbacks, updated_files, removed_files)
+      callbacks.each do |callback|
+        callback.call(updated_files, removed_files, self)
+      end
+    end
+  end
+end

middleman-core/lib/middleman-core/step_definitions/middleman_steps.rb

@@ -1,17 +1,9 @@
 Then /^the file "([^\"]*)" has the contents$/ do |path, contents|
   write_file(path, contents)
-  step %Q{the file "#{path}" did change}
+  @server_inst.files.find_new_files!
 end
 
 Then /^the file "([^\"]*)" is removed$/ do |path|
   step %Q{I remove the file "#{path}"}
-  step %Q{the file "#{path}" did delete}
+  @server_inst.files.find_new_files!
-end
-
-Then /^the file "([^\"]*)" did change$/ do |path|
-  @server_inst.files.did_change(path)
-end
-
-Then /^the file "([^\"]*)" did delete$/ do |path|
-  @server_inst.files.did_delete(path)
 end

middleman-core/lib/middleman-core/step_definitions/server_steps.rb

@@ -42,9 +42,11 @@ Given /^the Server is running$/ do
   ENV['MM_ROOT'] = root_dir
 
   initialize_commands = @initialize_commands || []
-  initialize_commands.unshift lambda { config[:show_exceptions] = false }
 
   @server_inst = ::Middleman::Application.new do
+    config[:watcher_disable] = true
+    config[:show_exceptions] = false
+
     initialize_commands.each do |p|
       instance_exec(&p)
     end

middleman-core/lib/middleman-core/template_context.rb

@@ -23,7 +23,7 @@ module Middleman
     attr_accessor :current_engine
 
     # Shorthand references to global values on the app instance.
-    def_delegators :@app, :config, :logger, :sitemap, :server?, :build?, :environment?, :data, :extensions, :source_dir, :root
+    def_delegators :@app, :config, :logger, :sitemap, :server?, :build?, :environment?, :data, :extensions, :root
 
     # Initialize a context with the current app and predefined locals and options hashes.
     #
@@ -64,10 +64,10 @@ module Middleman
       buf_was = save_buffer
 
       # Find a layout for this file
-      layout_path = ::Middleman::TemplateRenderer.locate_layout(@app, layout_name, current_engine)
+      layout_file = ::Middleman::TemplateRenderer.locate_layout(@app, layout_name, current_engine)
 
       # Get the layout engine
-      extension = File.extname(layout_path)
+      extension = File.extname(layout_file[:relative_path])
       engine = extension[1..-1].to_sym
 
       # Store last engine for later (could be inside nested renders)
@@ -84,7 +84,7 @@ module Middleman
         restore_buffer(buf_was)
       end
       # Render the layout, with the contents of the block inside.
-      concat_safe_content render_file(layout_path, @locs, @opts) { content }
+      concat_safe_content render_file(layout_file, @locs, @opts) { content }
     ensure
       # Reset engine back to template's value, regardless of success
       self.current_engine = engine_was
@@ -100,19 +100,19 @@ module Middleman
     def render(_, name, options={}, &block)
       name = name.to_s
 
-      partial_path = locate_partial(name)
+      partial_file = locate_partial(name)
 
-      raise ::Middleman::TemplateRenderer::TemplateNotFound, "Could not locate partial: #{name}" unless partial_path
+      raise ::Middleman::TemplateRenderer::TemplateNotFound, "Could not locate partial: #{name}" unless partial_file
 
-      r = sitemap.find_resource_by_path(sitemap.file_to_path(partial_path))
+      r = sitemap.find_resource_by_path(sitemap.file_to_path(partial_file))
 
       if r && !r.template?
-        File.read(r.source_file)
+        File.read(r.source_file[:full_path])
       else
         opts = options.dup
         locs = opts.delete(:locals)
 
-        render_file(partial_path, locs.freeze, opts.freeze, &block)
+        render_file(partial_file, locs.freeze, opts.freeze, &block)
       end
     end
 
@@ -123,43 +123,44 @@ module Middleman
     # @api private
     # @param [String] partial_path
     # @return [String]
-    Contract String => Maybe[String]
+    Contract String => Maybe[IsA['Middleman::SourceFile']]
     def locate_partial(partial_path)
       return unless resource = sitemap.find_resource_by_path(current_path)
 
       # Look for partials relative to the current path
-      current_dir = File.dirname(resource.source_file)
+      current_dir = resource.source_file[:relative_path].dirname
-      relative_dir = File.join(current_dir.sub(%r{^#{Regexp.escape(source_dir)}/?}, ''), partial_path)
+      non_root = partial_path.to_s.sub(/^\//, '')
+      relative_dir = current_dir + Pathname(non_root)
 
-      partial_path_no_underscore = partial_path.sub(/^_/, '').sub(/\/_/, '/')
+      non_root_no_underscore = non_root.sub(/^_/, '').sub(/\/_/, '/')
-      relative_dir_no_underscore = File.join(current_dir.sub(%r{^#{Regexp.escape(source_dir)}/?}, ''), partial_path_no_underscore)
+      relative_dir_no_underscore = current_dir + Pathname(non_root_no_underscore)
 
-      partial = nil
+      partial_file = nil
 
       [
-        [relative_dir, { preferred_engine: File.extname(resource.source_file)[1..-1].to_sym }],
+        [relative_dir.to_s, { preferred_engine: resource.source_file[:relative_path].extname[1..-1].to_sym }],
-        [File.join('', partial_path)],
+        [non_root],
-        [relative_dir_no_underscore, { try_static: true }],
+        [relative_dir_no_underscore.to_s, { try_static: true }],
-        [File.join('', partial_path_no_underscore), { try_static: true }]
+        [non_root_no_underscore, { try_static: true }]
       ].each do |args|
-        partial = ::Middleman::TemplateRenderer.resolve_template(@app, *args)
+        partial_file = ::Middleman::TemplateRenderer.resolve_template(@app, *args)
-        break if partial
+        break if partial_file
       end
 
-      partial
+      partial_file
     end
 
     # Render a path with locs, opts and contents block.
     #
     # @api private
-    # @param [String] path The file path.
+    # @param [Middleman::SourceFile] file The file.
     # @param [Hash] locs Template locals.
     # @param [Hash] opts Template options.
     # @param [Proc] block A block will be evaluated to return internal contents.
     # @return [String] The resulting content string.
-    Contract String, Hash, Hash, Proc => String
+    Contract IsA['Middleman::SourceFile'], Hash, Hash, Proc => String
-    def render_file(path, locs, opts, &block)
+    def render_file(file, locs, opts, &block)
-      file_renderer = ::Middleman::FileRenderer.new(@app, path)
+      file_renderer = ::Middleman::FileRenderer.new(@app, file[:relative_path].to_s)
       file_renderer.render(locs, opts, self, &block)
     end
 

middleman-core/lib/middleman-core/template_renderer.rb

@@ -65,8 +65,8 @@ module Middleman
       end
 
       # If we need a layout and have a layout, use it
-      if layout_path = fetch_layout(engine, options)
+      if layout_file = fetch_layout(engine, options)
-        layout_renderer = ::Middleman::FileRenderer.new(@app, layout_path)
+        layout_renderer = ::Middleman::FileRenderer.new(@app, layout_file[:relative_path].to_s)
         content = layout_renderer.render(locals, options, context) { content }
       end
 
@@ -85,11 +85,11 @@ module Middleman
     # @param [Symbol] engine
     # @param [Hash] opts
     # @return [String, Boolean]
-    Contract Symbol, Hash => Or[String, Bool]
+    Contract Symbol, Hash => Maybe[IsA['Middleman::SourceFile']]
     def fetch_layout(engine, opts)
       # The layout name comes from either the system default or the options
       local_layout = opts.key?(:layout) ? opts[:layout] : @app.config[:layout]
-      return false unless local_layout
+      return unless local_layout
 
       # Look for engine-specific options
       engine_options = @app.config.respond_to?(engine) ? @app.config.send(engine) : {}
@@ -108,12 +108,12 @@ module Middleman
       if local_layout == :_auto_layout
         # Look for :layout of any extension
         # If found, use it. If not, continue
-        locate_layout(:layout, layout_engine) || false
+        locate_layout(:layout, layout_engine)
       else
         # Look for specific layout
         # If found, use it. If not, error.
-        if layout_path = locate_layout(local_layout, layout_engine)
+        if layout_file = locate_layout(local_layout, layout_engine)
-          layout_path
+          layout_file
         else
           raise ::Middleman::TemplateRenderer::TemplateNotFound, "Could not locate layout: #{local_layout}"
         end
@@ -124,7 +124,7 @@ module Middleman
     # @param [String] name
     # @param [Symbol] preferred_engine
     # @return [String]
-    Contract Or[String, Symbol], Symbol => Maybe[String]
+    Contract Or[String, Symbol], Symbol => Maybe[IsA['Middleman::SourceFile']]
     def locate_layout(name, preferred_engine=nil)
       self.class.locate_layout(@app, name, preferred_engine)
     end
@@ -133,19 +133,19 @@ module Middleman
     # @param [String] name
     # @param [Symbol] preferred_engine
     # @return [String]
-    Contract IsA['Middleman::Application'], Or[String, Symbol], Symbol => Maybe[String]
+    Contract IsA['Middleman::Application'], Or[String, Symbol], Symbol => Maybe[IsA['Middleman::SourceFile']]
     def self.locate_layout(app, name, preferred_engine=nil)
       resolve_opts = {}
       resolve_opts[:preferred_engine] = preferred_engine unless preferred_engine.nil?
 
       # Check layouts folder
-      layout_path = resolve_template(app, File.join(app.config[:layouts_dir], name.to_s), resolve_opts)
+      layout_file = resolve_template(app, File.join(app.config[:layouts_dir], name.to_s), resolve_opts)
 
       # If we didn't find it, check root
-      layout_path = resolve_template(app, name, resolve_opts) unless layout_path
+      layout_file = resolve_template(app, name, resolve_opts) unless layout_file
 
       # Return the path
-      layout_path
+      layout_file
     end
 
     # Find a template on disk given a output path
@@ -161,72 +161,53 @@ module Middleman
     # @param [String] request_path
     # @option options [Boolean] :preferred_engine If set, try this engine first, then fall back to any engine.
     # @return [String, Boolean] Either the path to the template, or false
-    Contract IsA['Middleman::Application'], Or[Symbol, String], Maybe[Hash] => Maybe[String]
+    Contract IsA['Middleman::Application'], Or[Symbol, String], Maybe[Hash] => Maybe[IsA['Middleman::SourceFile']]
     def self.resolve_template(app, request_path, options={})
       # Find the path by searching or using the cache
-      request_path = request_path.to_s
+      relative_path = Util.strip_leading_slash(request_path.to_s)
 
       # Cache lookups in build mode only
       if app.build?
-        cache.fetch(:resolve_template, request_path, options) do
+        cache.fetch(:resolve_template, relative_path, options) do
-          uncached_resolve_template(app, request_path, options)
+          uncached_resolve_template(app, relative_path, options)
         end
       else
-        uncached_resolve_template(app, request_path, options)
+        uncached_resolve_template(app, relative_path, options)
       end
     end
 
-    Contract IsA['Middleman::Application'], String, Hash => Maybe[String]
+    Contract IsA['Middleman::Application'], Or[Symbol, String], Hash => Maybe[IsA['Middleman::SourceFile']]
-    def self.uncached_resolve_template(app, request_path, options)
+    def self.uncached_resolve_template(app, relative_path, options)
-      relative_path = Util.strip_leading_slash(request_path)
-      on_disk_path  = File.expand_path(relative_path, app.source_dir)
-
       # By default, any engine will do
-      preferred_engines = ['*']
+      preferred_engines = []
-      preferred_engines << nil if options[:try_static]
 
       # If we're specifically looking for a preferred engine
       if options.key?(:preferred_engine)
         extension_class = ::Tilt[options[:preferred_engine]]
 
         # Get a list of extensions for a preferred engine
-        matched_exts = ::Tilt.mappings.select do |_, engines|
+        preferred_engines += ::Tilt.mappings.select do |_, engines|
           engines.include? extension_class
         end.keys
-
-        # Prefer to look for the matched extensions
-        unless matched_exts.empty?
-          preferred_engines.unshift('{' + matched_exts.join(',') + '}')
-        end
       end
 
-      search_paths = preferred_engines.map do |preferred_engine|
+      preferred_engines << '*'
-        path_with_ext = on_disk_path.dup
+      preferred_engines << nil if options[:try_static]
+
+      found_template = nil
+
+      preferred_engines.each do |preferred_engine|
+        path_with_ext = relative_path.dup
         path_with_ext << ('.' + preferred_engine) unless preferred_engine.nil?
-        path_with_ext
-      end
 
-      found_path = nil
+        file = app.files.find(:source, path_with_ext, preferred_engine == '*')
-      search_paths.each do |path_with_ext|
-        found_path = Dir[path_with_ext].find do |path|
-          ::Tilt[path]
-        end
 
-        unless found_path
+        found_template = file if file && (preferred_engine.nil? || ::Tilt[file[:full_path]])
-          found_path = path_with_ext if File.exist?(path_with_ext)
+        break if found_template
-        end
-
-        break if found_path
       end
 
       # If we found one, return it
-      if found_path
+      found_template
-        found_path
-      elsif File.exist?(on_disk_path)
-        on_disk_path
-      else
-        nil
-      end
     end
   end
 end

middleman-core/lib/middleman-core/util.rb

@@ -24,9 +24,10 @@ module Middleman
     #
     # @param [String] filename The file to check.
     # @return [Boolean]
-    Contract String => Bool
+    Contract Or[String, Pathname] => Bool
     def binary?(filename)
-      ext = File.extname(filename)
+      path = Pathname(filename)
+      ext = path.extname
 
       # We hardcode detecting of gzipped SVG files
       return true if ext == '.svgz'
@@ -38,7 +39,7 @@ module Middleman
       if mime = ::Rack::Mime.mime_type(dot_ext, nil)
         !nonbinary_mime?(mime)
       else
-        file_contents_include_binary_bytes?(filename)
+        file_contents_include_binary_bytes?(path.to_s)
       end
     end
 
@@ -74,14 +75,15 @@ module Middleman
     # @private
     # @param [Hash] data Normal hash
     # @return [Middleman::Util::HashWithIndifferentAccess]
-    Contract Maybe[Or[Array, Hash, HashWithIndifferentAccess]] => Maybe[Frozen[Or[HashWithIndifferentAccess, Array]]]
+    FrozenDataStructure = Frozen[Or[HashWithIndifferentAccess, Array]]
+    Contract Maybe[Or[Array, Hash, HashWithIndifferentAccess]] => Maybe[FrozenDataStructure]
     def recursively_enhance(data)
       if data.is_a? HashWithIndifferentAccess
         data
       elsif data.is_a? Hash
         HashWithIndifferentAccess.new(data)
       elsif data.is_a? Array
-        data.map(&method(:recursively_enhance))
+        data.map(&method(:recursively_enhance)).freeze
       else
         nil
       end