diff --git a/CHANGELOG.md b/CHANGELOG.md index 22202562..9ae9a891 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ master === +* Removed ability to use JSON as frontmatter. Still allowed in data/ folder. +* Added YAML data postscript. Like frontmatter, but reversed. Attach content after the key/value data as a `:postscript` key to the data structure (if Hash). + # 4.0.0.beta.2 * Fixed regression causing exceptions to be silently thrown away outside of `--verbose` mode in the dev server. diff --git a/middleman-core/features/data.feature b/middleman-core/features/data.feature index fcce8445..db8bede6 100644 --- a/middleman-core/features/data.feature +++ b/middleman-core/features/data.feature @@ -51,3 +51,10 @@ Feature: Local Data API Then I should see "title1:Hello" Then I should see "title2:More" Then I should see "title3:Stuff" + + Scenario: Using data postscript + Given the Server is running at "nested-data-app" + When I go to "/extracontent.html" + Then I should see "

With Content

" + Then I should see '

Header 2

' + Then I should see "

Paragraph 1

" diff --git a/middleman-core/features/front-matter-neighbor.feature b/middleman-core/features/front-matter-neighbor.feature index 08b4f57c..e2e5cc69 100644 --- a/middleman-core/features/front-matter-neighbor.feature +++ b/middleman-core/features/front-matter-neighbor.feature @@ -30,17 +30,6 @@ Feature: Neighboring YAML Front Matter Then I should not see "---" When I go to "/front-matter-encoding.html.erb.frontmatter" Then I should see "File Not Found" - - Scenario: Rendering html (json) - Given the Server is running at "frontmatter-neighbor-app" - When I go to "/json-front-matter.html.erb.frontmatter" - Then I should see "File Not Found" - When I go to "/json-front-matter-2.php" - Then I should see "

This is the title

" - Then I should see "This is the title" Then I should not see "---" - - Scenario: Rendering html (json) - Given the Server is running at "frontmatter-app" - When I go to "/json-front-matter.html" - Then I should see "

This is the title

" - Then I should not see ";;;" - When I go to "/json-front-matter-2.php" - Then I should see "

This is the title

" - Then I should see "" - Then I should see ";;;" - - Scenario: JSON not on first line, with encoding - Given the Server is running at "frontmatter-app" - When I go to "/json-front-matter-encoding.html" - Then I should see "

This is the title

" - Then I should not see ";;;" Scenario: A template changes frontmatter during preview Given the Server is running at "frontmatter-app" diff --git a/middleman-core/fixtures/frontmatter-app/source/front-matter-line-2.html.erb b/middleman-core/fixtures/frontmatter-app/source/front-matter-line-2.html.erb index 7aeb16b9..d0c0802c 100644 --- a/middleman-core/fixtures/frontmatter-app/source/front-matter-line-2.html.erb +++ b/middleman-core/fixtures/frontmatter-app/source/front-matter-line-2.html.erb @@ -1,7 +1,10 @@

Test

+ +

<%= current_page.data.title %>

+ --- layout: false title: This is the title --- -

<%= current_page.data.title %>

+
Stuff
diff --git a/middleman-core/fixtures/frontmatter-app/source/json-front-matter-2.php.erb b/middleman-core/fixtures/frontmatter-app/source/json-front-matter-2.php.erb deleted file mode 100644 index 7208cf6f..00000000 --- a/middleman-core/fixtures/frontmatter-app/source/json-front-matter-2.php.erb +++ /dev/null @@ -1,7 +0,0 @@ -;;; - "layout": false, - "title": "This is the title" -;;; - -

<%= current_page.data.title %>

- diff --git a/middleman-core/fixtures/frontmatter-app/source/json-front-matter-encoding.html.erb b/middleman-core/fixtures/frontmatter-app/source/json-front-matter-encoding.html.erb deleted file mode 100644 index d05b1f70..00000000 --- a/middleman-core/fixtures/frontmatter-app/source/json-front-matter-encoding.html.erb +++ /dev/null @@ -1,7 +0,0 @@ -# encoding: UTF-8 -;;; -"layout": false, -"title": "This is the title" -;;; - -

<%= current_page.data.title %>

diff --git a/middleman-core/fixtures/frontmatter-app/source/json-front-matter-line-2.html.erb b/middleman-core/fixtures/frontmatter-app/source/json-front-matter-line-2.html.erb deleted file mode 100644 index 56b65add..00000000 --- a/middleman-core/fixtures/frontmatter-app/source/json-front-matter-line-2.html.erb +++ /dev/null @@ -1,7 +0,0 @@ -

Test

-;;; -layout: false, -title: "This is the title" -;;; - -

<%= current_page.data.title %>

diff --git a/middleman-core/fixtures/frontmatter-app/source/json-front-matter.html.erb b/middleman-core/fixtures/frontmatter-app/source/json-front-matter.html.erb deleted file mode 100644 index da001783..00000000 --- a/middleman-core/fixtures/frontmatter-app/source/json-front-matter.html.erb +++ /dev/null @@ -1,6 +0,0 @@ -;;; - "layout": false, - "title": "This is the title" -;;; - -

<%= current_page.data.title %>

diff --git a/middleman-core/fixtures/frontmatter-neighbor-app/config.rb b/middleman-core/fixtures/frontmatter-neighbor-app/config.rb index f5edde7f..babfbc2c 100644 --- a/middleman-core/fixtures/frontmatter-neighbor-app/config.rb +++ b/middleman-core/fixtures/frontmatter-neighbor-app/config.rb @@ -15,7 +15,7 @@ class NeighborFrontmatter < ::Middleman::Extension next unless file - fmdata = app.extensions[:front_matter].frontmatter_and_content(file[:full_path]).first + fmdata = ::Middleman::Util::Data.parse(file[:full_path], :yaml).first opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type) opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options) ignored = fmdata.delete(:ignored) diff --git a/middleman-core/fixtures/frontmatter-neighbor-app/source/json-front-matter-2.php.erb b/middleman-core/fixtures/frontmatter-neighbor-app/source/json-front-matter-2.php.erb deleted file mode 100644 index c271a92f..00000000 --- a/middleman-core/fixtures/frontmatter-neighbor-app/source/json-front-matter-2.php.erb +++ /dev/null @@ -1,2 +0,0 @@ -

<%= current_page.data.title %>

- diff --git a/middleman-core/fixtures/frontmatter-neighbor-app/source/json-front-matter-2.php.erb.frontmatter b/middleman-core/fixtures/frontmatter-neighbor-app/source/json-front-matter-2.php.erb.frontmatter deleted file mode 100644 index 415cefbc..00000000 --- a/middleman-core/fixtures/frontmatter-neighbor-app/source/json-front-matter-2.php.erb.frontmatter +++ /dev/null @@ -1,4 +0,0 @@ -;;; - "layout": false, - "title": "This is the title" -;;; \ No newline at end of file diff --git a/middleman-core/fixtures/frontmatter-neighbor-app/source/json-front-matter.html.erb b/middleman-core/fixtures/frontmatter-neighbor-app/source/json-front-matter.html.erb deleted file mode 100644 index 1ae95c52..00000000 --- a/middleman-core/fixtures/frontmatter-neighbor-app/source/json-front-matter.html.erb +++ /dev/null @@ -1 +0,0 @@ -

<%= current_page.data.title %>

diff --git a/middleman-core/fixtures/frontmatter-neighbor-app/source/json-front-matter.html.erb.frontmatter b/middleman-core/fixtures/frontmatter-neighbor-app/source/json-front-matter.html.erb.frontmatter deleted file mode 100644 index 415cefbc..00000000 --- a/middleman-core/fixtures/frontmatter-neighbor-app/source/json-front-matter.html.erb.frontmatter +++ /dev/null @@ -1,4 +0,0 @@ -;;; - "layout": false, - "title": "This is the title" -;;; \ No newline at end of file diff --git a/middleman-core/fixtures/frontmatter-settings-neighbor-app/config.rb b/middleman-core/fixtures/frontmatter-settings-neighbor-app/config.rb index 011d5230..1fc88608 100644 --- a/middleman-core/fixtures/frontmatter-settings-neighbor-app/config.rb +++ b/middleman-core/fixtures/frontmatter-settings-neighbor-app/config.rb @@ -26,7 +26,7 @@ class NeighborFrontmatter < ::Middleman::Extension end def apply_neighbor_data(resource, file) - fmdata = app.extensions[:front_matter].frontmatter_and_content(file[:full_path]).first + fmdata = ::Middleman::Util::Data.parse(file[:full_path], :yaml).first opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type) opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options) ignored = fmdata.delete(:ignored) diff --git a/middleman-core/fixtures/nested-data-app/data/examples/withcontent.yaml b/middleman-core/fixtures/nested-data-app/data/examples/withcontent.yaml new file mode 100644 index 00000000..74d583cd --- /dev/null +++ b/middleman-core/fixtures/nested-data-app/data/examples/withcontent.yaml @@ -0,0 +1,11 @@ +--- +name: "With Content" +--- + +## Header 2 + +Paragraph 1 + +Paragraph 2 + +### Header 3 \ No newline at end of file diff --git a/middleman-core/fixtures/nested-data-app/source/extracontent.html.haml.erb b/middleman-core/fixtures/nested-data-app/source/extracontent.html.haml.erb new file mode 100644 index 00000000..90e0e6f8 --- /dev/null +++ b/middleman-core/fixtures/nested-data-app/source/extracontent.html.haml.erb @@ -0,0 +1,4 @@ +%h1= data.examples.withcontent.name + +:markdown + <%= data.examples.withcontent.postscript.gsub("\n", "\n\s\s") %> diff --git a/middleman-core/lib/middleman-core/configuration.rb b/middleman-core/lib/middleman-core/configuration.rb index e8c9b647..81422461 100644 --- a/middleman-core/lib/middleman-core/configuration.rb +++ b/middleman-core/lib/middleman-core/configuration.rb @@ -33,7 +33,6 @@ module Middleman # Set the value of a setting by key. Creates the setting if it doesn't exist. # @param [Symbol] key # @param [Object] val - # rubocop:disable UselessSetterCall def []=(key, val) setting_obj = setting(key) || define_setting(key) setting_obj.value = val diff --git a/middleman-core/lib/middleman-core/core_extensions/data.rb b/middleman-core/lib/middleman-core/core_extensions/data.rb index 205021d4..9169410a 100644 --- a/middleman-core/lib/middleman-core/core_extensions/data.rb +++ b/middleman-core/lib/middleman-core/core_extensions/data.rb @@ -1,6 +1,5 @@ -require 'yaml' -require 'active_support/json' require 'middleman-core/contracts' +require 'middleman-core/util/data' module Middleman module CoreExtensions @@ -100,9 +99,10 @@ module Middleman basename = File.basename(data_path, extension) if %w(.yaml .yml).include?(extension) - data = ::YAML.load_file(file[:full_path]) + data, postscript = ::Middleman::Util::Data.parse(file[:full_path], :yaml) + data[:postscript] = postscript if !postscript.nil? && data.is_a?(Hash) elsif extension == '.json' - data = ::ActiveSupport::JSON.decode(file[:full_path].read) + data, _postscript = ::Middleman::Util::Data.parse(file[:full_path], :json) else return end diff --git a/middleman-core/lib/middleman-core/core_extensions/front_matter.rb b/middleman-core/lib/middleman-core/core_extensions/front_matter.rb index 9f672a7e..0e26259a 100644 --- a/middleman-core/lib/middleman-core/core_extensions/front_matter.rb +++ b/middleman-core/lib/middleman-core/core_extensions/front_matter.rb @@ -1,11 +1,12 @@ -require 'active_support/core_ext/hash/keys' +# Core Pathname library used for traversal require 'pathname' -# Parsing YAML frontmatter -require 'yaml' +# DbC +require 'middleman-core/contracts' -# Parsing JSON frontmatter -require 'active_support/json' +require 'active_support/core_ext/hash/keys' + +require 'middleman-core/util/data' # Extensions namespace module Middleman::CoreExtensions @@ -13,13 +14,6 @@ module Middleman::CoreExtensions # Try to run after routing but before directory_indexes self.resource_list_manipulator_priority = 90 - YAML_ERRORS = [StandardError] - - # https://github.com/tenderlove/psych/issues/23 - if defined?(Psych) && defined?(Psych::SyntaxError) - YAML_ERRORS << Psych::SyntaxError - end - def initialize(app, options_hash={}, &block) super @@ -71,7 +65,7 @@ module Middleman::CoreExtensions return [{}, nil] unless file - @cache[file[:full_path]] ||= frontmatter_and_content(file[:full_path]) + @cache[file[:full_path]] ||= ::Middleman::Util::Data.parse(file[:full_path]) end Contract ArrayOf[IsA['Middleman::SourceFile']], ArrayOf[IsA['Middleman::SourceFile']] => Any @@ -80,93 +74,5 @@ module Middleman::CoreExtensions @cache.delete(file[:full_path]) end end - - # Get the frontmatter and plain content from a file - # @param [String] path - # @return [Array] - Contract Pathname => [Hash, Maybe[String]] - def frontmatter_and_content(full_path) - data = {} - - return [data, nil] if ::Middleman::Util.binary?(full_path) - - # Avoid weird race condition when a file is renamed. - content = begin - File.read(full_path) - rescue ::EOFError - rescue ::IOError - rescue ::Errno::ENOENT - '' - end - - begin - if content =~ /\A.*coding:/ - lines = content.split(/\n/) - lines.shift - content = lines.join("\n") - end - - result = parse_yaml_front_matter(content, full_path) || parse_json_front_matter(content, full_path) - return result if result - rescue - # Probably a binary file, move on - end - - [data, content] - end - - private - - # Parse YAML frontmatter out of a string - # @param [String] content - # @return [Array] - Contract String, Pathname => Maybe[[Hash, String]] - def parse_yaml_front_matter(content, full_path) - yaml_regex = /\A(---\s*\n.*?\n?)^((---|\.\.\.)\s*$\n?)/m - if content =~ yaml_regex - content = content.sub(yaml_regex, '') - - begin - data = YAML.load($1) || {} - data = data.symbolize_keys - rescue *YAML_ERRORS => e - app.logger.error "YAML Exception parsing #{full_path}: #{e.message}" - return nil - end - else - return nil - end - - [data, content] - rescue - [{}, content] - end - - # Parse JSON frontmatter out of a string - # @param [String] content - # @return [Array] - Contract String, Pathname => Maybe[[Hash, String]] - def parse_json_front_matter(content, full_path) - json_regex = /\A(;;;\s*\n.*?\n?)^(;;;\s*$\n?)/m - - if content =~ json_regex - content = content.sub(json_regex, '') - - begin - json = ($1 + $2).sub(';;;', '{').sub(';;;', '}') - data = ::ActiveSupport::JSON.decode(json).symbolize_keys - rescue => e - app.logger.error "JSON Exception parsing #{full_path}: #{e.message}" - return nil - end - - else - return nil - end - - [data, content] - rescue - [{}, content] - end end end diff --git a/middleman-core/lib/middleman-core/renderers/liquid.rb b/middleman-core/lib/middleman-core/renderers/liquid.rb index 16817caf..e09306b4 100644 --- a/middleman-core/lib/middleman-core/renderers/liquid.rb +++ b/middleman-core/lib/middleman-core/renderers/liquid.rb @@ -27,7 +27,26 @@ module Middleman next unless resource.source_file[:relative_path].to_s =~ %r{\.liquid$} # Convert data object into a hash for liquid - resource.add_metadata locals: { data: app.extensions[:data].data_store.to_h } + resource.add_metadata locals: { + data: stringify_recursive(app.extensions[:data].data_store.to_h) + } + end + end + + def stringify_recursive(hash) + {}.tap do |h| + hash.each { |key, value| h[key.to_s] = map_value(value) } + end + end + + def map_value(thing) + case thing + when Hash + stringify_recursive(thing) + when Array + thing.map { |v| map_value(v) } + else + thing end end end diff --git a/middleman-core/lib/middleman-core/util/data.rb b/middleman-core/lib/middleman-core/util/data.rb new file mode 100644 index 00000000..1d107ef9 --- /dev/null +++ b/middleman-core/lib/middleman-core/util/data.rb @@ -0,0 +1,153 @@ +# Core Pathname library used for traversal +require 'pathname' + +# DbC +require 'middleman-core/contracts' + +# Shared util methods +require 'middleman-core/util' + +# Parsing YAML data +require 'yaml' + +# Parsing JSON data +require 'active_support/json' + +module Middleman + module Util + module Data + include Contracts + + module_function + + YAML_ERRORS = [StandardError] + + # https://github.com/tenderlove/psych/issues/23 + if defined?(Psych) && defined?(Psych::SyntaxError) + YAML_ERRORS << Psych::SyntaxError + end + + # Get the frontmatter and plain content from a file + # @param [String] path + # @return [Array] + Contract Pathname, Maybe[Symbol] => [Hash, Maybe[String]] + def parse(full_path, known_type=nil) + data = {} + + return [data, nil] if ::Middleman::Util.binary?(full_path) + + # Avoid weird race condition when a file is renamed. + content = begin + File.read(full_path) + rescue ::EOFError + rescue ::IOError + rescue ::Errno::ENOENT + '' + end + + begin + if content =~ /\A.*coding:/ + lines = content.split(/\n/) + lines.shift + content = lines.join("\n") + end + + if known_type + if known_type == :yaml + result = parse_yaml(content, full_path, true) + elsif known_type == :json + result = parse_json(content, full_path) + end + else + result = parse_yaml(content, full_path, false) + end + + return result if result + rescue + # Probably a binary file, move on + end + + [data, content] + end + + # Parse YAML frontmatter out of a string + # @param [String] content + # @return [Array] + Contract String, Pathname, Bool => Maybe[[Hash, String]] + def parse_yaml(content, full_path, require_yaml=false) + total_delims = content.scan(/^(?:---|\.\.\.)\s*(?:\n|$)/).length + has_first_line_delim = !content.match(/\A(---\s*(?:\n|$))/).nil? + # has_closing_delim = (total_delims > 1 && has_first_line_delim) || (!has_first_line_delim && total_delims == 1) + + parts = content.split(/^(?:---|\.\.\.)\s*(?:\n|$)/) + parts.shift if parts[0].empty? + + yaml_string = nil + additional_content = nil + + if require_yaml + yaml_string = parts[0] + additional_content = parts[1] + else + if total_delims > 1 + if has_first_line_delim + yaml_string = parts[0] + additional_content = parts[1] + else + additional_content = content + end + else + additional_content = parts[0] + end + end + + return [{}, additional_content] if yaml_string.nil? + + begin + data = map_value(::YAML.load(yaml_string) || {}) + rescue *YAML_ERRORS => e + $stderr.puts "YAML Exception parsing #{full_path}: #{e.message}" + return nil + end + + [data, additional_content] + rescue + [{}, additional_content] + end + + # Parse JSON frontmatter out of a string + # @param [String] content + # @return [Array] + Contract String, Pathname => Maybe[[Hash, String]] + def parse_json(content, full_path) + begin + data = map_value(::ActiveSupport::JSON.decode(content)) + rescue => e + $stderr.puts "JSON Exception parsing #{full_path}: #{e.message}" + return nil + end + + [data, nil] + rescue + [{}, nil] + end + + def symbolize_recursive(hash) + {}.tap do |h| + hash.each { |key, value| h[key.to_sym] = map_value(value) } + end + end + + def map_value(thing) + case thing + when Hash + symbolize_recursive(thing) + when Array + thing.map { |v| map_value(v) } + else + thing + end + end + end + end +end