Merge pull request #650 from middleman/sitemap_query

Make Sitemap metadata queryable with arel-style API

middleman-core/features/queryable.feature

@@ -0,0 +1,31 @@
+Feature: Queryable Selector
+  Scenario: Basic Selector Tests
+    Then should initialize with an attribute and an operator
+    Then should raise an exception if the operator is not supported
+  Scenario: Using offset and limit
+    Given the Server is running at "queryable-app"
+    Then should limit the documents to the number specified
+    Then should offset the documents by the number specified
+    Then should support offset and limit at the same time
+    Then should not freak out about an offset higher than the document count
+  Scenario: Using where queries with an equal operator
+    Given the Server is running at "queryable-app"
+    Then should return the right documents
+    Then should be chainable
+    Then should not be confused by attributes not present in all documents
+  Scenario: Using where queries with a complex operator
+    Given the Server is running at "queryable-app"
+    Then with a gt operator should return the right documents
+    Then with a gte operator should return the right documents
+    Then with an in operator should return the right documents
+    Then with an lt operator should return the right documents
+    Then with an lte operator should return the right documents
+    Then with an include operator include should return the right documents
+    Then with mixed operators should return the right documents
+    Then using multiple constrains in one where should return the right documents
+  Scenario: Sorting documents
+    Given the Server is running at "queryable-app"
+    Then should support ordering by attribute ascending
+    Then should support ordering by attribute descending
+    Then should order by attribute ascending by default
+    Then should exclude documents that do not own the attribute

middleman-core/features/step_definitions/queryable_steps.rb

@@ -0,0 +1,123 @@
+Then /^should initialize with an attribute and an operator$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :author, :operator => 'equal'
+  :author.should == selector.attribute
+  'equal'.should == selector.operator
+end
+
+Then /^should raise an exception if the operator is not supported$/ do
+  expect {
+    selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :author, :operator => 'zomg'
+  }.to raise_error(::Middleman::Sitemap::Queryable::OperatorNotSupportedError)
+end
+
+Then /^should limit the documents to the number specified$/ do
+  @server_inst.sitemap.order_by(:id).limit(2).all.map { |r| r.raw_data[:id] }.sort.should == [1,2].sort
+end
+
+Then /^should offset the documents by the number specified$/ do
+  @server_inst.sitemap.order_by(:id).offset(2).all.map { |r| r.raw_data[:id] }.sort.should == [3,4,5].sort
+end
+
+Then /^should support offset and limit at the same time$/ do
+  @server_inst.sitemap.order_by(:id).offset(1).limit(2).all.map { |r| r.raw_data[:id] }.sort.should == [2,3].sort
+end
+
+Then /^should not freak out about an offset higher than the document count$/ do
+  @server_inst.sitemap.order_by(:id).offset(5).all.should == []
+end
+
+Then /^should return the right documents$/ do
+  documents = @server_inst.sitemap.resources.select { |r| !r.raw_data.empty? }
+  document_1 = documents[0]
+  document_2 = documents[1]
+
+  found_document = @server_inst.sitemap.where(:title => document_1.raw_data[:title]).first
+  document_1.should == found_document
+  
+  found_document = @server_inst.sitemap.where(:title => document_2.raw_data[:title]).first
+  document_2.should == found_document
+end
+
+Then /^should be chainable$/ do
+  documents = @server_inst.sitemap.resources.select { |r| !r.raw_data.empty? }
+  document_1 = documents[0]
+
+  document_proxy = @server_inst.sitemap.where(:title => document_1.raw_data[:title])
+  document_proxy.where(:id => document_1.raw_data[:id])
+  document_1.should == document_proxy.first
+end
+
+Then /^should not be confused by attributes not present in all documents$/ do
+  result = @server_inst.sitemap.where(:seldom_attribute => 'is seldom').all
+  result.map { |r| r.raw_data[:id] }.should == [4]
+end
+
+Then /^with a gt operator should return the right documents$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'gt'
+  found_documents = @server_inst.sitemap.where(selector => 2).all
+  found_documents.map { |r| r.raw_data[:id] }.sort.should == [5,3,4].sort
+end
+
+Then /^with a gte operator should return the right documents$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'gte'
+  found_documents = @server_inst.sitemap.where(selector => 2).all
+  found_documents.map { |r| r.raw_data[:id] }.sort.should == [2,5,3,4].sort
+end
+
+Then /^with an in operator should return the right documents$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'in'
+  found_documents = @server_inst.sitemap.where(selector => [2,3]).all
+  found_documents.map { |r| r.raw_data[:id] }.sort.should == [2,3].sort
+end
+
+Then /^with an lt operator should return the right documents$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'lt'
+  found_documents = @server_inst.sitemap.where(selector => 2).all
+  found_documents.map { |r| r.raw_data[:id] }.should == [1]
+end
+
+Then /^with an lte operator should return the right documents$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'lte'
+  found_documents = @server_inst.sitemap.where(selector => 2).all
+  found_documents.map { |r| r.raw_data[:id] }.sort.should == [1,2].sort
+end
+
+Then /^with an include operator include should return the right documents$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :tags, :operator => 'include'
+  found_documents = @server_inst.sitemap.where(selector => 'ruby').all
+  found_documents.map { |r| r.raw_data[:id] }.sort.should == [1,2].sort
+end
+
+Then /^with mixed operators should return the right documents$/ do
+  in_selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'in'
+  gt_selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'gt'
+  documents_proxy = @server_inst.sitemap.where(in_selector => [2,3])
+  found_documents = documents_proxy.where(gt_selector => 2).all
+  found_documents.map { |r| r.raw_data[:id] }.should == [3]
+end
+
+Then /^using multiple constrains in one where should return the right documents$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'lte'
+  found_documents = @server_inst.sitemap.where(selector => 2, :status => :published).all
+  found_documents.map { |r| r.raw_data[:id] }.sort.should == [1,2].sort
+end
+
+Then /^should support ordering by attribute ascending$/ do
+  found_documents = @server_inst.sitemap.order_by(:title => :asc).all
+  found_documents.map { |r| r.raw_data[:id] }.should == [2,3,1,5,4]
+end
+
+Then /^should support ordering by attribute descending$/ do
+  found_documents = @server_inst.sitemap.order_by(:title => :desc).all
+  found_documents.map { |r| r.raw_data[:id] }.should == [4,5,1,3,2]
+end
+
+Then /^should order by attribute ascending by default$/ do
+  found_documents = @server_inst.sitemap.order_by(:title).all
+  found_documents.map { |r| r.raw_data[:id] }.should == [2,3,1,5,4]
+end
+
+Then /^should exclude documents that do not own the attribute$/ do
+  found_documents = @server_inst.sitemap.order_by(:status).all
+  found_documents.map { |r| r.raw_data[:id] }.to_set.should == [1,2].to_set
+end

middleman-core/fixtures/queryable-app/source/2010-08-08-test-document-file.html.markdown

@@ -0,0 +1,8 @@
+---
+id: 1
+title: Some fancy title
+tags: [ruby]
+status: :published
+---
+
+I like being the demo text.

middleman-core/fixtures/queryable-app/source/2010-08-09-another-test-document.html.markdown

@@ -0,0 +1,10 @@
+---
+id: 2
+title: Another title, that's for sure
+tags: [ruby, rails]
+special_attribute: Yes!
+friends: [Anton, Paul]
+status: :published
+---
+
+The body copy.

middleman-core/fixtures/queryable-app/source/2011-12-26-some-test-document.html.markdown

@@ -0,0 +1,6 @@
+---
+id: 5
+title: Some test document
+---
+
+This is just some test document.

middleman-core/fixtures/queryable-app/source/document_with_date_in_yaml.html.markdown

@@ -0,0 +1,7 @@
+---
+id: 3
+title: Document with date in YAML
+date: 2011-04-05
+---
+
+This document has no date in the filename, but in the YAML front matter.

middleman-core/fixtures/queryable-app/source/document_without_date.html.markdown

@@ -0,0 +1,7 @@
+---
+id: 4
+title: This document has no date
+seldom_attribute: is seldom
+---
+
+This document has no date at all.

middleman-core/lib/middleman-core/core_extensions/front_matter.rb

@@ -1,3 +1,5 @@
+require "active_support/core_ext/hash/keys"
+
 # Extensions namespace
 module Middleman::CoreExtensions
 
@@ -34,8 +36,8 @@ module Middleman::CoreExtensions
             fmdata = frontmatter_manager.data(path).first || {}
 
             data = {}
-            %w(layout layout_engine).each do |opt|
+            [:layout, :layout_engine].each do |opt|
-              data[opt.to_sym] = fmdata[opt] unless fmdata[opt].nil?
+              data[opt] = fmdata[opt] unless fmdata[opt].nil?
             end
 
             { :options => data, :page => fmdata }
@@ -85,7 +87,7 @@ module Middleman::CoreExtensions
           content = content.sub(yaml_regex, "")
 
           begin
-            data = YAML.load($1)
+            data = YAML.load($1).symbolize_keys
           rescue *YAML_ERRORS => e
             logger.error "YAML Exception: #{e.message}"
             return false
@@ -108,7 +110,7 @@ module Middleman::CoreExtensions
 
           begin
             json = ($1+$2).sub(";;;", "{").sub(";;;", "}")
-            data = ActiveSupport::JSON.decode(json)
+            data = ActiveSupport::JSON.decode(json).symbolize_keys
           rescue => e
             logger.error "JSON Exception: #{e.message}"
             return false
@@ -147,7 +149,7 @@ module Middleman::CoreExtensions
           # Probably a binary file, move on
         end
 
-        [::Middleman::Util.recursively_enhance(data).freeze, content]
+        [data, content]
       end
 
       def normalize_path(path)
@@ -187,10 +189,20 @@ module Middleman::CoreExtensions
 
       # This page's frontmatter
       # @return [Hash]
-      def data
+      def raw_data
         app.frontmatter_manager.data(source_file).first
       end
 
+      def data
+        @_last_raw ||= nil
+        @_last_enhanced ||= nil
+
+        if @_last_raw != raw_data
+          @_last_raw == raw_data
+          @_last_enhanced = ::Middleman::Util.recursively_enhance(raw_data).freeze
+        end
+      end
+
     end
 
     module InstanceMethods

middleman-core/lib/middleman-core/sitemap/queryable.rb

@@ -0,0 +1,148 @@
+require "active_support/core_ext/object/inclusion"
+
+module Middleman
+  module Sitemap
+    
+    # Code adapted from https://github.com/ralph/document_mapper/
+    module Queryable
+      OPERATOR_MAPPING = {
+        'equal'   => :==,
+        'gt'      => :>,
+        'gte'     => :>=,
+        'in'      => :in?,
+        'include' => :include?,
+        'lt'      => :<,
+        'lte'     => :<=
+      }
+
+      VALID_OPERATORS = OPERATOR_MAPPING.keys
+
+      FileNotFoundError = Class.new StandardError
+      OperatorNotSupportedError = Class.new StandardError
+
+      module API
+        def select(options = {})
+          documents = resources.select { |r| !r.raw_data.empty? }
+          options[:where].each do |selector, selector_value|
+            documents = documents.select do |document|
+              next unless document.raw_data.has_key? selector.attribute
+              document_value = document.raw_data[selector.attribute]
+              operator = OPERATOR_MAPPING[selector.operator]
+              document_value.send operator, selector_value
+            end
+          end
+
+          if options[:order_by].present?
+            order_attribute = options[:order_by].keys.first
+            asc_or_desc = options[:order_by].values.first
+            documents = documents.select do |document|
+              document.raw_data.include? order_attribute
+            end
+            documents = documents.sort_by do |document|
+              document.raw_data[order_attribute]
+            end
+            documents.reverse! if asc_or_desc == :desc
+          end
+
+          documents
+        end
+
+        def where(hash)
+          Query.new(self).where(hash)
+        end
+
+        def order_by(field)
+          Query.new(self).order_by(field)
+        end
+
+        def offset(number)
+          Query.new(self).offset(number)
+        end
+
+        def limit(number)
+          Query.new(self).limit(number)
+        end
+      end
+
+      class Query
+        def initialize(model)
+          @model = model
+          @where = {}
+        end
+
+        def where(constraints_hash)
+          selector_hash = constraints_hash.reject { |key, value| !key.is_a? Selector }
+          symbol_hash = constraints_hash.reject { |key, value| key.is_a? Selector }
+          symbol_hash.each do |attribute, value|
+            selector = Selector.new(:attribute => attribute, :operator => 'equal')
+            selector_hash.update({ selector => value })
+          end
+          @where.merge! selector_hash
+          self
+        end
+
+        def order_by(field)
+          @order_by = field.is_a?(Symbol) ? {field => :asc} : field
+          self
+        end
+
+        def offset(number)
+          @offset = number
+          self
+        end
+
+        def limit(number)
+          @limit = number
+          self
+        end
+
+        def first
+          self.all.first
+        end
+
+        def last
+          self.all.last
+        end
+
+        def all
+          result = @model.select(:where => @where, :order_by => @order_by)
+          if @offset.present?
+            result = result.last([result.size - @offset, 0].max)
+          end
+          if @limit.present?
+            result = result.first(@limit)
+          end
+          result
+        end
+      end
+
+      class Selector
+        attr_reader :attribute, :operator
+
+        def initialize(opts = {})
+          unless VALID_OPERATORS.include? opts[:operator]
+            raise OperatorNotSupportedError
+          end
+          @attribute, @operator = opts[:attribute], opts[:operator]
+        end
+      end
+    end
+  end
+end
+
+# Add operators to symbol objects
+class Symbol
+  Middleman::Sitemap::Queryable::VALID_OPERATORS.each do |operator|
+    class_eval <<-OPERATORS
+      def #{operator}
+        Middleman::Sitemap::Queryable::Selector.new(:attribute => self, :operator => '#{operator}')
+      end
+    OPERATORS
+  end
+  
+  unless method_defined?(:"<=>")
+    def <=>(other)
+      self.to_s <=> other.to_s
+    end
+  end
+end

middleman-core/lib/middleman-core/sitemap/store.rb

@@ -1,6 +1,7 @@
 # Used for merging results of metadata callbacks
 require "active_support/core_ext/hash/deep_merge"
 require 'monitor'
+require "middleman-core/sitemap/queryable"
 
 module Middleman
 
@@ -18,6 +19,8 @@ module Middleman
       # @return [Middleman::Application]
       attr_accessor :app
 
+      include ::Middleman::Sitemap::Queryable::API
+
       # Initialize with parent app
       # @param [Middleman::Application] app
       def initialize(app)

middleman-more/lib/middleman-more/extensions/directory_indexes.rb

@@ -40,8 +40,8 @@ module Middleman
                     File.extname(index_file) != resource.ext
 
             # Check if frontmatter turns directory_index off
-            d = resource.data
+            d = resource.raw_data
-            next if d && d["directory_index"] == false
+            next if d && d[:directory_index] == false
 
             # Check if file metadata (options set by "page" in config.rb) turns directory_index off
             if resource.metadata[:options] && resource.metadata[:options][:directory_index] == false