Merge: Get local mode passing against chef-pedant

.travis.yml

@@ -11,11 +11,15 @@ before_install:
   - gem update --system 2.1.11
   - gem --version
 
-rvm:
-  - 1.8.7
-  - 1.9.3
-  - 2.0.0
-  - 2.1.0
+matrix:
+  include:
+    - rvm: 1.8.7
+    - rvm: 1.9.3
+    - rvm: 2.0.0
+    - rvm: 2.1.0
+    - rvm: 2.1.0
+      gemfile: pedant.gemfile
+      script: bundle exec rake pedant
 
 branches:
   only:

Rakefile

@@ -48,6 +48,10 @@ task :ship => :gem do
   end
 end
 
+task :pedant do
+  require File.expand_path('spec/support/pedant/run_pedant')
+end
+
 begin
   require 'yard'
   DOC_FILES = [ "README.rdoc", "LICENSE", "spec/tiny_server.rb", "lib/**/*.rb" ]
@@ -63,5 +67,3 @@ begin
 rescue LoadError
   puts "yard is not available. (sudo) gem install yard to generate yard documentation."
 end
-
-

lib/chef/chef_fs/chef_fs_data_store.rb

@@ -23,6 +23,7 @@
 require 'chef/chef_fs/file_system'
 require 'chef/chef_fs/file_system/not_found_error'
 require 'chef/chef_fs/file_system/memory_root'
+require 'fileutils'
 
 class Chef
   module ChefFS
@@ -43,7 +44,11 @@ def create_dir(path, name, *options)
           @memory_store.create_dir(path, name, *options)
         else
           with_dir(path) do |parent|
-            parent.create_child(chef_fs_filename(path + [name]), nil)
+            begin
+              parent.create_child(chef_fs_filename(path + [name]), nil)
+            rescue Chef::ChefFS::FileSystem::AlreadyExistsError => e
+              raise ChefZero::DataStore::DataAlreadyExistsError.new(to_zero_path(e.entry), e)
+            end
           end
         end
       end
@@ -61,7 +66,11 @@ def create(path, name, data, *options)
           end
 
           with_dir(path) do |parent|
-            parent.create_child(chef_fs_filename(path + [name]), data)
+            begin
+              parent.create_child(chef_fs_filename(path + [name]), data)
+            rescue Chef::ChefFS::FileSystem::AlreadyExistsError => e
+              raise ChefZero::DataStore::DataAlreadyExistsError.new(to_zero_path(e.entry), e)
+            end
           end
         end
       end
@@ -82,7 +91,13 @@ def get(path, request=nil)
           with_entry(path) do |entry|
             if path[0] == 'cookbooks' && path.length == 3
               # get /cookbooks/NAME/version
-              result = entry.chef_object.to_hash
+              result = nil
+              begin
+                result = entry.chef_object.to_hash
+              rescue Chef::ChefFS::FileSystem::NotFoundError => e
+                raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
+              end
+
               result.each_pair do |key, value|
                 if value.is_a?(Array)
                   value.each do |file|
@@ -102,7 +117,11 @@ def get(path, request=nil)
               JSON.pretty_generate(result)
 
             else
-              entry.read
+              begin
+                entry.read
+              rescue Chef::ChefFS::FileSystem::NotFoundError => e
+                raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
+              end
             end
           end
         end
@@ -121,7 +140,12 @@ def set(path, data, *options)
             write_cookbook(path, data, *options)
           else
             with_dir(path[0..-2]) do |parent|
-              parent.create_child(chef_fs_filename(path), data)
+              child = parent.child(chef_fs_filename(path))
+              if child.exists?
+                child.write(data)
+              else
+                parent.create_child(chef_fs_filename(path), data)
+              end
             end
           end
         end
@@ -132,10 +156,14 @@ def delete(path)
           @memory_store.delete(path)
         else
           with_entry(path) do |entry|
-            if path[0] == 'cookbooks' && path.length >= 3
-              entry.delete(true)
-            else
-              entry.delete(false)
+            begin
+              if path[0] == 'cookbooks' && path.length >= 3
+                entry.delete(true)
+              else
+                entry.delete(false)
+              end
+            rescue Chef::ChefFS::FileSystem::NotFoundError => e
+              raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
             end
           end
         end
@@ -146,7 +174,11 @@ def delete_dir(path, *options)
           @memory_store.delete_dir(path, *options)
         else
           with_entry(path) do |entry|
-            entry.delete(options.include?(:recursive))
+            begin
+              entry.delete(options.include?(:recursive))
+            rescue Chef::ChefFS::FileSystem::NotFoundError => e
+              raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
+            end
           end
         end
       end
@@ -172,10 +204,16 @@ def list(path)
 
         elsif path[0] == 'cookbooks' && path.length == 2
           if Chef::Config.versioned_cookbooks
-            # list /cookbooks/name = filter /cookbooks/name-version down to name
-            entry.children.map { |child| split_name_version(child.name) }.
-                           select { |name, version| name == path[1] }.
-                           map { |name, version| version }.to_a
+            result = with_entry([ 'cookbooks' ]) do |entry|
+              # list /cookbooks/name = filter /cookbooks/name-version down to name
+              entry.children.map { |child| split_name_version(child.name) }.
+                             select { |name, version| name == path[1] }.
+                             map { |name, version| version }
+            end
+            if result.empty?
+              raise ChefZero::DataStore::DataNotFoundError.new(path)
+            end
+            result
           else
             # list /cookbooks/name = <single version>
             version = get_single_cookbook_version(path)
@@ -186,12 +224,12 @@ def list(path)
           with_entry(path) do |entry|
             begin
               entry.children.map { |c| zero_filename(c) }.sort
-            rescue Chef::ChefFS::FileSystem::NotFoundError
+            rescue Chef::ChefFS::FileSystem::NotFoundError => e
               # /cookbooks, /data, etc. never return 404
               if path_always_exists?(path)
                 []
               else
-                raise
+                raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
               end
             end
           end
@@ -223,28 +261,36 @@ def use_memory_store?(path)
       end
 
       def write_cookbook(path, data, *options)
-        # Create a little Chef::ChefFS memory filesystem with the data
         if Chef::Config.versioned_cookbooks
-          cookbook_path = "cookbooks/#{path[1]}-#{path[2]}"
+          cookbook_path = File.join('cookbooks', "#{path[1]}-#{path[2]}")
         else
-          cookbook_path = "cookbooks/#{path[1]}"
+          cookbook_path = File.join('cookbooks', path[1])
         end
+
+        # Create a little Chef::ChefFS memory filesystem with the data
         cookbook_fs = Chef::ChefFS::FileSystem::MemoryRoot.new('uploading')
         cookbook = JSON.parse(data, :create_additions => false)
         cookbook.each_pair do |key, value|
           if value.is_a?(Array)
             value.each do |file|
               if file.is_a?(Hash) && file.has_key?('checksum')
                 file_data = @memory_store.get(['file_store', 'checksums', file['checksum']])
-                cookbook_fs.add_file("#{cookbook_path}/#{file['path']}", file_data)
+                cookbook_fs.add_file(File.join(cookbook_path, file['path']), file_data)
               end
             end
           end
         end
 
-        # Use the copy/diff algorithm to copy it down so we don't destroy
-        # chefignored data.  This is terribly un-thread-safe.
-        Chef::ChefFS::FileSystem.copy_to(Chef::ChefFS::FilePattern.new("/#{cookbook_path}"), cookbook_fs, chef_fs, nil, {:purge => true})
+        # Create the .uploaded-cookbook-version.json
+        cookbooks = chef_fs.child('cookbooks')
+        if !cookbooks.exists?
+          cookbooks = chef_fs.create_child('cookbooks')
+        end
+        # We are calling a cookbooks-specific API, so get multiplexed_dirs out of the way if it is there
+        if cookbooks.respond_to?(:multiplexed_dirs)
+          cookbooks = cookbooks.write_dir
+        end
+        cookbooks.write_cookbook(cookbook_path, data, cookbook_fs)
       end
 
       def split_name_version(entry_name)
@@ -343,8 +389,10 @@ def with_entry(path)
       end
 
       def with_dir(path)
+        # Do not automatically create data bags
+        create = !(path[0] == 'data' && path.size >= 2)
         begin
-          yield get_dir(_to_chef_fs_path(path), true)
+          yield get_dir(_to_chef_fs_path(path), create)
         rescue Chef::ChefFS::FileSystem::NotFoundError => e
           raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
         end

lib/chef/chef_fs/file_system/chef_repository_file_system_cookbook_dir.rb

@@ -44,11 +44,17 @@ def chef_object
             end
 
             loader.load_cookbooks
-            return loader.cookbook_version
-          rescue
-            Chef::Log.error("Could not read #{path_for_printing} into a Chef object: #{$!}")
+            cb = loader.cookbook_version
+            if !cb
+              Chef::Log.error("Cookbook #{file_path} empty.")
+              raise "Cookbook #{file_path} empty."
+            end
+            cb
+          rescue => e
+            Chef::Log.error("Could not read #{path_for_printing} into a Chef object: #{e}")
+            Chef::Log.error(e.backtrace.join("\n"))
+            raise
           end
-          nil
         end
 
         def children
@@ -66,6 +72,8 @@ def can_have_child?(name, is_dir)
           if is_dir
             # Only the given directories will be uploaded.
             return CookbookDir::COOKBOOK_SEGMENT_INFO.keys.include?(name.to_sym) && name != 'root_files'
+          elsif name == Chef::Cookbook::CookbookVersionLoader::UPLOADED_COOKBOOK_VERSION_FILE
+            return false
           end
           super(name, is_dir)
         end
@@ -81,6 +89,14 @@ def canonical_cookbook_name(entry_name)
           self.class.canonical_cookbook_name(entry_name)
         end
 
+        def uploaded_cookbook_version_path
+          File.join(file_path, Chef::Cookbook::CookbookVersionLoader::UPLOADED_COOKBOOK_VERSION_FILE)
+        end
+
+        def can_upload?
+          File.exists?(uploaded_cookbook_version_path) || children.size > 0
+        end
+
         protected
 
         def make_child(child_name)

lib/chef/chef_fs/file_system/chef_repository_file_system_cookbooks_dir.rb

@@ -43,7 +43,7 @@ def children
                 map { |child_name| make_child(child_name) }.
                 select do |entry|
                   # empty cookbooks and cookbook directories are ignored
-                  if entry.children.size == 0
+                  if !entry.can_upload?
                     Chef::Log.warn("Cookbook '#{entry.name}' is empty or entirely chefignored at #{entry.path_for_printing}")
                     false
                   else
@@ -59,6 +59,25 @@ def can_have_child?(name, is_dir)
           is_dir && !name.start_with?('.')
         end
 
+        def write_cookbook(cookbook_path, cookbook_version_json, from_fs)
+          cookbook_name = File.basename(cookbook_path)
+          child = make_child(cookbook_name)
+
+          # Use the copy/diff algorithm to copy it down so we don't destroy
+          # chefignored data.  This is terribly un-thread-safe.
+          Chef::ChefFS::FileSystem.copy_to(Chef::ChefFS::FilePattern.new("/#{cookbook_path}"), from_fs, child, nil, {:purge => true})
+
+          # Write out .uploaded-cookbook-version.json
+          cookbook_file_path = File.join(file_path, cookbook_name)
+          if !File.exists?(cookbook_file_path)
+            FileUtils.mkdir_p(cookbook_file_path)
+          end
+          uploaded_cookbook_version_path = File.join(cookbook_file_path, Chef::Cookbook::CookbookVersionLoader::UPLOADED_COOKBOOK_VERSION_FILE)
+          File.open(uploaded_cookbook_version_path, 'w') do |file|
+            file.write(cookbook_version_json)
+          end
+        end
+
         protected
 
         def make_child(child_name)

lib/chef/chef_fs/file_system/file_system_entry.rb

@@ -18,8 +18,9 @@
 
 require 'chef/chef_fs/file_system/base_fs_dir'
 require 'chef/chef_fs/file_system/rest_list_dir'
-require 'chef/chef_fs/file_system/not_found_error'
+require 'chef/chef_fs/file_system/already_exists_error'
 require 'chef/chef_fs/file_system/must_delete_recursively_error'
+require 'chef/chef_fs/file_system/not_found_error'
 require 'chef/chef_fs/path_utils'
 require 'fileutils'
 
@@ -48,15 +49,18 @@ def children
 
         def create_child(child_name, file_contents=nil)
           child = make_child(child_name)
+          if child.exists?
+            raise Chef::ChefFS::FileSystem::AlreadyExistsError.new(:create_child, child)
+          end
           if file_contents
             child.write(file_contents)
           else
             begin
               Dir.mkdir(child.file_path)
             rescue Errno::EEXIST
+              raise Chef::ChefFS::FileSystem::AlreadyExistsError.new(:create_child, child)
             end
           end
-          @children = nil
           child
         end
 
@@ -75,6 +79,10 @@ def delete(recurse)
           end
         end
 
+        def exists?
+          File.exists?(file_path)
+        end
+
         def read
           begin
             File.open(file_path, "rb") {|f| f.read}