Merge pull request #408 from bhollis/unicode

Support Unicode characters in filenames

middleman-core/features/dynamic_pages.feature

@@ -21,6 +21,7 @@ Feature: Dynamic Pages
       | target_ignore2.html                           |
       | target_ignore3.html                           |
       | target_ignore4.html                           |
+      | 明日がある.html                               |
     Then the following files should not exist:
       | should_be_ignored.html                        |
       | should_be_ignored2.html                       |

middleman-core/features/unicode_filenames.feature

@@ -0,0 +1,14 @@
+Feature: Unicode filenames
+  In order to support non-ASCII characters in filenames
+
+  Scenario: Build with files containing unicode characters in their name
+    Given a successfully built app at "unicode-filenames-app"
+    When I cd to "build"
+    Then the file "snowmen-rule-☃.html" should contain "☃"
+
+  Scenario: Preview with files containing unicode characters in their name
+    Given the Server is running at "unicode-filenames-app"
+    When I go to "/snowmen-rule-☃.html"
+    # There seem to be encoding issues w/ the test framework so we can't check the content
+    Then I should see "Snowman!"
+    

middleman-core/fixtures/dynamic-pages-app/config.rb

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 page "/fake.html", :proxy => "/real.html", :layout => false
 page "fake2.html", :proxy => "/real.html", :layout => false
 page "fake3.html", :proxy => "real.html", :layout => false
@@ -28,6 +29,8 @@ page "/target_ignore4.html", :proxy => "should_be_ignored8.html", :ignore => tru
   end
 end
 
+page "明日がある.html", :proxy => "/real.html", :layout => false
+
 page "f*/*" do
   @all_glob = "I am all glob"
 end

middleman-core/fixtures/unicode-filenames-app/config.rb

@@ -0,0 +1 @@
+

middleman-core/fixtures/unicode-filenames-app/source/snowmen-rule-☃.html

@@ -0,0 +1,4 @@
+Snowman!
+<div style="text-align:center; font-size:4000%;">
+  ☃
+</div>

middleman-core/lib/middleman-core/application.rb

@@ -13,6 +13,10 @@ require "middleman-core/vendor/hooks-0.2.0/lib/hooks"
 
 require "middleman-core/sitemap"
   
+# Let's serve all HTML as UTF-8
+::Rack::Mime::MIME_TYPES['.html'] = 'text/html;charset=utf8'
+::Rack::Mime::MIME_TYPES['.htm'] = 'text/html;charset=utf8'
+
 # Core Middleman Class
 module Middleman
   class Application
@@ -381,18 +385,20 @@ module Middleman
       start_time = Time.now
 
       # Normalize the path and add index if we're looking at a directory
-      @original_path = env["PATH_INFO"].dup
-      @escaped_path  = @original_path.gsub("%20", " ")
-      @request_path  = full_path(@escaped_path)
-    
+      @original_path = URI.decode(env["PATH_INFO"].dup)
+      if @original_path.respond_to? :force_encoding
+        @original_path.force_encoding('UTF-8')
+      end
+      @request_path  = full_path(@original_path)
+
       # Run before callbacks
       run_hook :before
 
-      if @escaped_path != @request_path
+      if @original_path != @request_path
         # Get the resource object for this path
-        resource = sitemap.find_resource_by_destination_path(@escaped_path)
+        resource = sitemap.find_resource_by_destination_path(@original_path)
       end
-    
+
       # Get the resource object for this full path
       resource ||= sitemap.find_resource_by_destination_path(@request_path)
     
@@ -407,7 +413,7 @@ module Middleman
       
       # Set a HTTP content type based on the request's extensions
       content_type resource.mime_type
-    
+      
       begin
         # Write out the contents of the page
         res.write resource.render

middleman-core/lib/middleman-core/cli/build.rb

@@ -104,7 +104,8 @@ module Middleman::Cli
       output_file = File.join(build_dir, resource.destination_path)
 
       begin
-        response = self.class.shared_rack.get(resource.destination_path.gsub(/\s/, "%20"))
+        response = self.class.shared_rack.get(URI.escape(resource.destination_path))
+
         if response.status == 200
           create_file(output_file, response.body, { :force => true })
         else
@@ -201,7 +202,7 @@ module Middleman::Cli
       @app.sitemap.resources.select do |resource|
         resource.ext == ".css"
       end.each do |resource|
-        Middleman::Cli::Build.shared_rack.get(resource.destination_path.gsub(/\s/, "%20"))
+        Middleman::Cli::Build.shared_rack.get(URI.escape(resource.destination_path))
       end
       
       puts "== Checking for Compass sprites" if @app.logging?

middleman-core/lib/middleman-core/step_definitions/server_steps.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 require "rack/test"
 
 Given /^a clean server$/ do
@@ -61,11 +63,11 @@ Given /^the Server is running at "([^\"]*)"$/ do |app_path|
 end
 
 When /^I go to "([^\"]*)"$/ do |url|
-  @browser.get(url)
+  @browser.get(URI.escape(url))
 end
 
 Then /^going to "([^\"]*)" should not raise an exception$/ do |url|
-  lambda { @browser.get(url) }.should_not raise_exception
+  lambda { @browser.get(URI.escape(url)) }.should_not raise_exception
 end
 
 Then /^I should see "([^\"]*)"$/ do |expected|

middleman-core/lib/middleman-core/util.rb

@@ -29,7 +29,8 @@ module Middleman
     # @param [String] path
     # @return [String]
     def self.normalize_path(path)
-      path.sub(/^\//, "").gsub("%20", " ")
+      # The tr call works around a bug in Ruby's Unicode handling
+      path.sub(/^\//, "").tr('','') 
     end
 
     # Extract the text of a Rack response as a string.