Merge pull request #650 from middleman/sitemap_query

Make Sitemap metadata queryable with arel-style API
This commit is contained in:
Thomas Reynolds 2012-12-25 16:06:22 -08:00
commit ad1806ccc0
12 changed files with 363 additions and 8 deletions

View file

@ -0,0 +1,31 @@
Feature: Queryable Selector
Scenario: Basic Selector Tests
Then should initialize with an attribute and an operator
Then should raise an exception if the operator is not supported
Scenario: Using offset and limit
Given the Server is running at "queryable-app"
Then should limit the documents to the number specified
Then should offset the documents by the number specified
Then should support offset and limit at the same time
Then should not freak out about an offset higher than the document count
Scenario: Using where queries with an equal operator
Given the Server is running at "queryable-app"
Then should return the right documents
Then should be chainable
Then should not be confused by attributes not present in all documents
Scenario: Using where queries with a complex operator
Given the Server is running at "queryable-app"
Then with a gt operator should return the right documents
Then with a gte operator should return the right documents
Then with an in operator should return the right documents
Then with an lt operator should return the right documents
Then with an lte operator should return the right documents
Then with an include operator include should return the right documents
Then with mixed operators should return the right documents
Then using multiple constrains in one where should return the right documents
Scenario: Sorting documents
Given the Server is running at "queryable-app"
Then should support ordering by attribute ascending
Then should support ordering by attribute descending
Then should order by attribute ascending by default
Then should exclude documents that do not own the attribute

View file

@ -0,0 +1,123 @@
Then /^should initialize with an attribute and an operator$/ do
selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :author, :operator => 'equal'
:author.should == selector.attribute
'equal'.should == selector.operator
end
Then /^should raise an exception if the operator is not supported$/ do
expect {
selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :author, :operator => 'zomg'
}.to raise_error(::Middleman::Sitemap::Queryable::OperatorNotSupportedError)
end
Then /^should limit the documents to the number specified$/ do
@server_inst.sitemap.order_by(:id).limit(2).all.map { |r| r.raw_data[:id] }.sort.should == [1,2].sort
end
Then /^should offset the documents by the number specified$/ do
@server_inst.sitemap.order_by(:id).offset(2).all.map { |r| r.raw_data[:id] }.sort.should == [3,4,5].sort
end
Then /^should support offset and limit at the same time$/ do
@server_inst.sitemap.order_by(:id).offset(1).limit(2).all.map { |r| r.raw_data[:id] }.sort.should == [2,3].sort
end
Then /^should not freak out about an offset higher than the document count$/ do
@server_inst.sitemap.order_by(:id).offset(5).all.should == []
end
Then /^should return the right documents$/ do
documents = @server_inst.sitemap.resources.select { |r| !r.raw_data.empty? }
document_1 = documents[0]
document_2 = documents[1]
found_document = @server_inst.sitemap.where(:title => document_1.raw_data[:title]).first
document_1.should == found_document
found_document = @server_inst.sitemap.where(:title => document_2.raw_data[:title]).first
document_2.should == found_document
end
Then /^should be chainable$/ do
documents = @server_inst.sitemap.resources.select { |r| !r.raw_data.empty? }
document_1 = documents[0]
document_proxy = @server_inst.sitemap.where(:title => document_1.raw_data[:title])
document_proxy.where(:id => document_1.raw_data[:id])
document_1.should == document_proxy.first
end
Then /^should not be confused by attributes not present in all documents$/ do
result = @server_inst.sitemap.where(:seldom_attribute => 'is seldom').all
result.map { |r| r.raw_data[:id] }.should == [4]
end
Then /^with a gt operator should return the right documents$/ do
selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'gt'
found_documents = @server_inst.sitemap.where(selector => 2).all
found_documents.map { |r| r.raw_data[:id] }.sort.should == [5,3,4].sort
end
Then /^with a gte operator should return the right documents$/ do
selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'gte'
found_documents = @server_inst.sitemap.where(selector => 2).all
found_documents.map { |r| r.raw_data[:id] }.sort.should == [2,5,3,4].sort
end
Then /^with an in operator should return the right documents$/ do
selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'in'
found_documents = @server_inst.sitemap.where(selector => [2,3]).all
found_documents.map { |r| r.raw_data[:id] }.sort.should == [2,3].sort
end
Then /^with an lt operator should return the right documents$/ do
selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'lt'
found_documents = @server_inst.sitemap.where(selector => 2).all
found_documents.map { |r| r.raw_data[:id] }.should == [1]
end
Then /^with an lte operator should return the right documents$/ do
selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'lte'
found_documents = @server_inst.sitemap.where(selector => 2).all
found_documents.map { |r| r.raw_data[:id] }.sort.should == [1,2].sort
end
Then /^with an include operator include should return the right documents$/ do
selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :tags, :operator => 'include'
found_documents = @server_inst.sitemap.where(selector => 'ruby').all
found_documents.map { |r| r.raw_data[:id] }.sort.should == [1,2].sort
end
Then /^with mixed operators should return the right documents$/ do
in_selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'in'
gt_selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'gt'
documents_proxy = @server_inst.sitemap.where(in_selector => [2,3])
found_documents = documents_proxy.where(gt_selector => 2).all
found_documents.map { |r| r.raw_data[:id] }.should == [3]
end
Then /^using multiple constrains in one where should return the right documents$/ do
selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'lte'
found_documents = @server_inst.sitemap.where(selector => 2, :status => :published).all
found_documents.map { |r| r.raw_data[:id] }.sort.should == [1,2].sort
end
Then /^should support ordering by attribute ascending$/ do
found_documents = @server_inst.sitemap.order_by(:title => :asc).all
found_documents.map { |r| r.raw_data[:id] }.should == [2,3,1,5,4]
end
Then /^should support ordering by attribute descending$/ do
found_documents = @server_inst.sitemap.order_by(:title => :desc).all
found_documents.map { |r| r.raw_data[:id] }.should == [4,5,1,3,2]
end
Then /^should order by attribute ascending by default$/ do
found_documents = @server_inst.sitemap.order_by(:title).all
found_documents.map { |r| r.raw_data[:id] }.should == [2,3,1,5,4]
end
Then /^should exclude documents that do not own the attribute$/ do
found_documents = @server_inst.sitemap.order_by(:status).all
found_documents.map { |r| r.raw_data[:id] }.to_set.should == [1,2].to_set
end

View file

@ -0,0 +1,8 @@
---
id: 1
title: Some fancy title
tags: [ruby]
status: :published
---
I like being the demo text.

View file

@ -0,0 +1,10 @@
---
id: 2
title: Another title, that's for sure
tags: [ruby, rails]
special_attribute: Yes!
friends: [Anton, Paul]
status: :published
---
The body copy.

View file

@ -0,0 +1,6 @@
---
id: 5
title: Some test document
---
This is just some test document.

View file

@ -0,0 +1,7 @@
---
id: 3
title: Document with date in YAML
date: 2011-04-05
---
This document has no date in the filename, but in the YAML front matter.

View file

@ -0,0 +1,7 @@
---
id: 4
title: This document has no date
seldom_attribute: is seldom
---
This document has no date at all.

View file

@ -1,3 +1,5 @@
require "active_support/core_ext/hash/keys"
# Extensions namespace # Extensions namespace
module Middleman::CoreExtensions module Middleman::CoreExtensions
@ -34,8 +36,8 @@ module Middleman::CoreExtensions
fmdata = frontmatter_manager.data(path).first || {} fmdata = frontmatter_manager.data(path).first || {}
data = {} data = {}
%w(layout layout_engine).each do |opt| [:layout, :layout_engine].each do |opt|
data[opt.to_sym] = fmdata[opt] unless fmdata[opt].nil? data[opt] = fmdata[opt] unless fmdata[opt].nil?
end end
{ :options => data, :page => fmdata } { :options => data, :page => fmdata }
@ -85,7 +87,7 @@ module Middleman::CoreExtensions
content = content.sub(yaml_regex, "") content = content.sub(yaml_regex, "")
begin begin
data = YAML.load($1) data = YAML.load($1).symbolize_keys
rescue *YAML_ERRORS => e rescue *YAML_ERRORS => e
logger.error "YAML Exception: #{e.message}" logger.error "YAML Exception: #{e.message}"
return false return false
@ -108,7 +110,7 @@ module Middleman::CoreExtensions
begin begin
json = ($1+$2).sub(";;;", "{").sub(";;;", "}") json = ($1+$2).sub(";;;", "{").sub(";;;", "}")
data = ActiveSupport::JSON.decode(json) data = ActiveSupport::JSON.decode(json).symbolize_keys
rescue => e rescue => e
logger.error "JSON Exception: #{e.message}" logger.error "JSON Exception: #{e.message}"
return false return false
@ -147,7 +149,7 @@ module Middleman::CoreExtensions
# Probably a binary file, move on # Probably a binary file, move on
end end
[::Middleman::Util.recursively_enhance(data).freeze, content] [data, content]
end end
def normalize_path(path) def normalize_path(path)
@ -187,10 +189,20 @@ module Middleman::CoreExtensions
# This page's frontmatter # This page's frontmatter
# @return [Hash] # @return [Hash]
def data def raw_data
app.frontmatter_manager.data(source_file).first app.frontmatter_manager.data(source_file).first
end end
def data
@_last_raw ||= nil
@_last_enhanced ||= nil
if @_last_raw != raw_data
@_last_raw == raw_data
@_last_enhanced = ::Middleman::Util.recursively_enhance(raw_data).freeze
end
end
end end
module InstanceMethods module InstanceMethods

View file

@ -0,0 +1,148 @@
require "active_support/core_ext/object/inclusion"
module Middleman
module Sitemap
# Code adapted from https://github.com/ralph/document_mapper/
module Queryable
OPERATOR_MAPPING = {
'equal' => :==,
'gt' => :>,
'gte' => :>=,
'in' => :in?,
'include' => :include?,
'lt' => :<,
'lte' => :<=
}
VALID_OPERATORS = OPERATOR_MAPPING.keys
FileNotFoundError = Class.new StandardError
OperatorNotSupportedError = Class.new StandardError
module API
def select(options = {})
documents = resources.select { |r| !r.raw_data.empty? }
options[:where].each do |selector, selector_value|
documents = documents.select do |document|
next unless document.raw_data.has_key? selector.attribute
document_value = document.raw_data[selector.attribute]
operator = OPERATOR_MAPPING[selector.operator]
document_value.send operator, selector_value
end
end
if options[:order_by].present?
order_attribute = options[:order_by].keys.first
asc_or_desc = options[:order_by].values.first
documents = documents.select do |document|
document.raw_data.include? order_attribute
end
documents = documents.sort_by do |document|
document.raw_data[order_attribute]
end
documents.reverse! if asc_or_desc == :desc
end
documents
end
def where(hash)
Query.new(self).where(hash)
end
def order_by(field)
Query.new(self).order_by(field)
end
def offset(number)
Query.new(self).offset(number)
end
def limit(number)
Query.new(self).limit(number)
end
end
class Query
def initialize(model)
@model = model
@where = {}
end
def where(constraints_hash)
selector_hash = constraints_hash.reject { |key, value| !key.is_a? Selector }
symbol_hash = constraints_hash.reject { |key, value| key.is_a? Selector }
symbol_hash.each do |attribute, value|
selector = Selector.new(:attribute => attribute, :operator => 'equal')
selector_hash.update({ selector => value })
end
@where.merge! selector_hash
self
end
def order_by(field)
@order_by = field.is_a?(Symbol) ? {field => :asc} : field
self
end
def offset(number)
@offset = number
self
end
def limit(number)
@limit = number
self
end
def first
self.all.first
end
def last
self.all.last
end
def all
result = @model.select(:where => @where, :order_by => @order_by)
if @offset.present?
result = result.last([result.size - @offset, 0].max)
end
if @limit.present?
result = result.first(@limit)
end
result
end
end
class Selector
attr_reader :attribute, :operator
def initialize(opts = {})
unless VALID_OPERATORS.include? opts[:operator]
raise OperatorNotSupportedError
end
@attribute, @operator = opts[:attribute], opts[:operator]
end
end
end
end
end
# Add operators to symbol objects
class Symbol
Middleman::Sitemap::Queryable::VALID_OPERATORS.each do |operator|
class_eval <<-OPERATORS
def #{operator}
Middleman::Sitemap::Queryable::Selector.new(:attribute => self, :operator => '#{operator}')
end
OPERATORS
end
unless method_defined?(:"<=>")
def <=>(other)
self.to_s <=> other.to_s
end
end
end

View file

@ -1,6 +1,7 @@
# Used for merging results of metadata callbacks # Used for merging results of metadata callbacks
require "active_support/core_ext/hash/deep_merge" require "active_support/core_ext/hash/deep_merge"
require 'monitor' require 'monitor'
require "middleman-core/sitemap/queryable"
module Middleman module Middleman
@ -18,6 +19,8 @@ module Middleman
# @return [Middleman::Application] # @return [Middleman::Application]
attr_accessor :app attr_accessor :app
include ::Middleman::Sitemap::Queryable::API
# Initialize with parent app # Initialize with parent app
# @param [Middleman::Application] app # @param [Middleman::Application] app
def initialize(app) def initialize(app)

View file

@ -40,8 +40,8 @@ module Middleman
File.extname(index_file) != resource.ext File.extname(index_file) != resource.ext
# Check if frontmatter turns directory_index off # Check if frontmatter turns directory_index off
d = resource.data d = resource.raw_data
next if d && d["directory_index"] == false next if d && d[:directory_index] == false
# Check if file metadata (options set by "page" in config.rb) turns directory_index off # Check if file metadata (options set by "page" in config.rb) turns directory_index off
if resource.metadata[:options] && resource.metadata[:options][:directory_index] == false if resource.metadata[:options] && resource.metadata[:options][:directory_index] == false