Merge branch 'jk/cookbook-artifacts'

12 files changed, 214 insertions, 23 deletions

diff --git a/chef-config/lib/chef-config/config.rb b/chef-config/lib/chef-config/config.rb
index 81e9cf3f68..5705ffbf56 100644
--- a/chef-config/lib/chef-config/config.rb
+++ b/chef-config/lib/chef-config/config.rb
@@ -144,13 +144,20 @@ module ChefConfig
 
     # Location of acls on disk. String or array of strings.
     # Defaults to <chef_repo_path>/acls.
-    # Only applies to Enterprise Chef commands.
     default(:acl_path) { derive_path_from_chef_repo_path("acls") }
 
     # Location of clients on disk. String or array of strings.
     # Defaults to <chef_repo_path>/acls.
     default(:client_path) { derive_path_from_chef_repo_path("clients") }
 
+    # Location of containers on disk. String or array of strings.
+    # Defaults to <chef_repo_path>/containers.
+    default(:container_path) { derive_path_from_chef_repo_path("containers") }
+
+    # Location of cookbook_artifacts on disk. String or array of strings.
+    # Defaults to <chef_repo_path>/cookbook_artifacts.
+    default(:cookbook_artifact_path) { derive_path_from_chef_repo_path("cookbook_artifacts") }
+
     # Location of cookbooks on disk. String or array of strings.
     # Defaults to <chef_repo_path>/cookbooks.  If chef_repo_path
     # is not specified, this is set to [/var/chef/cookbooks, /var/chef/site-cookbooks]).
@@ -163,11 +170,6 @@ module ChefConfig
       end
     end
 
-    # Location of containers on disk. String or array of strings.
-    # Defaults to <chef_repo_path>/containers.
-    # Only applies to Enterprise Chef commands.
-    default(:container_path) { derive_path_from_chef_repo_path("containers") }
-
     # Location of data bags on disk. String or array of strings.
     # Defaults to <chef_repo_path>/data_bags.
     default(:data_bag_path) { derive_path_from_chef_repo_path("data_bags") }
@@ -178,7 +180,6 @@ module ChefConfig
 
     # Location of groups on disk. String or array of strings.
     # Defaults to <chef_repo_path>/groups.
-    # Only applies to Enterprise Chef commands.
     default(:group_path) { derive_path_from_chef_repo_path("groups") }
 
     # Location of nodes on disk. String or array of strings.
@@ -199,7 +200,6 @@ module ChefConfig
 
     # Location of users on disk. String or array of strings.
     # Defaults to <chef_repo_path>/users.
-    # Does not apply to Enterprise Chef commands.
     default(:user_path) { derive_path_from_chef_repo_path("users") }
 
     # Location of policies on disk. String or array of strings.
diff --git a/lib/chef/chef_fs/config.rb b/lib/chef/chef_fs/config.rb
index dbba396ebb..5b9d34441b 100644
--- a/lib/chef/chef_fs/config.rb
+++ b/lib/chef/chef_fs/config.rb
@@ -33,6 +33,7 @@ class Chef
         "acls" => "acl",
         "clients" => "client",
         "cookbooks" => "cookbook",
+        "cookbook_artifacts" => "cookbook_artifact",
         "containers" => "container",
         "data_bags" => "data_bag",
         "environments" => "environment",
@@ -235,7 +236,7 @@ class Chef
           when "static"
             object_names = %w(cookbooks data_bags environments roles)
           when "hosted_everything"
-            object_names = %w(acls clients cookbooks containers data_bags environments groups nodes roles policies policy_groups)
+            object_names = %w(acls clients cookbooks cookbook_artifacts containers data_bags environments groups nodes roles policies policy_groups)
           else
             object_names = %w(clients cookbooks data_bags environments nodes roles users)
           end
diff --git a/lib/chef/chef_fs/file_system/chef_server/chef_server_root_dir.rb b/lib/chef/chef_fs/file_system/chef_server/chef_server_root_dir.rb
index 4a45d218d1..969cea876e 100644
--- a/lib/chef/chef_fs/file_system/chef_server/chef_server_root_dir.rb
+++ b/lib/chef/chef_fs/file_system/chef_server/chef_server_root_dir.rb
@@ -21,6 +21,7 @@ require "chef/chef_fs/file_system/chef_server/acls_dir"
 require "chef/chef_fs/file_system/base_fs_dir"
 require "chef/chef_fs/file_system/chef_server/rest_list_dir"
 require "chef/chef_fs/file_system/chef_server/cookbooks_dir"
+require "chef/chef_fs/file_system/chef_server/cookbook_artifacts_dir"
 require "chef/chef_fs/file_system/chef_server/versioned_cookbooks_dir"
 require "chef/chef_fs/file_system/chef_server/data_bags_dir"
 require "chef/chef_fs/file_system/chef_server/nodes_dir"
@@ -158,6 +159,8 @@ class Chef
                   RestListDir.new("clients", self, nil, Chef::ChefFS::DataHandler::ClientDataHandler.new),
                   # /containers
                   RestListDir.new("containers", self, nil, Chef::ChefFS::DataHandler::ContainerDataHandler.new),
+                  # /cookbook_artifacts
+                  CookbookArtifactsDir.new("cookbook_artifacts", self),
                   # /groups
                   RestListDir.new("groups", self, nil, Chef::ChefFS::DataHandler::GroupDataHandler.new),
                   # /nodes
diff --git a/lib/chef/chef_fs/file_system/chef_server/cookbook_artifact_dir.rb b/lib/chef/chef_fs/file_system/chef_server/cookbook_artifact_dir.rb
new file mode 100644
index 0000000000..ff3cde51a8
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/chef_server/cookbook_artifact_dir.rb
@@ -0,0 +1,38 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/chef_server/cookbook_dir'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      module ChefServer
+        class CookbookArtifactDir < CookbookDir
+          def initialize(name, parent, options = {})
+            super(name, parent)
+            @cookbook_name, dash, @version = name.rpartition('-')
+          end
+
+          def copy_from(other, options = {})
+            raise OperationNotAllowedError.new(:write, self, nil, "cannot be updated: cookbook artifacts are immutable once uploaded")
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/chef_server/cookbook_artifacts_dir.rb b/lib/chef/chef_fs/file_system/chef_server/cookbook_artifacts_dir.rb
new file mode 100644
index 0000000000..b001c28dc4
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/chef_server/cookbook_artifacts_dir.rb
@@ -0,0 +1,102 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/chef_server/cookbooks_dir'
+require 'chef/chef_fs/file_system/chef_server/cookbook_artifact_dir'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      module ChefServer
+        #
+        # /cookbook_artifacts
+        #
+        # Example children of /cookbook_artifacts:
+        #
+        # - apache2-ab234098245908ddf324a
+        # - apache2-295387a9823745feff239
+        # - mysql-1a2b9e1298734dfe90444
+        #
+        class CookbookArtifactsDir < CookbooksDir
+
+          def make_child_entry(name)
+            result = @children.select { |child| child.name == name }.first if @children
+            result || CookbookArtifactDir.new(name, self)
+          end
+
+          def children
+            @children ||= begin
+              result = []
+              root.get_json("#{api_path}/?num_versions=all").each_pair do |cookbook_name, cookbooks|
+                cookbooks['versions'].each do |cookbook_version|
+                  result << CookbookArtifactDir.new("#{cookbook_name}-#{cookbook_version['identifier']}", self)
+                end
+              end
+              result.sort_by(&:name)
+            end
+          end
+
+          # Knife currently does not understand versioned cookbooks
+          # Cookbook Version uploader also requires a lot of refactoring
+          # to make this work. So instead, we make a temporary cookbook
+          # symlinking back to real cookbook, and upload the proxy.
+          def upload_cookbook(other, options)
+            cookbook_name, dash, identifier = other.name.rpartition('-')
+
+            Dir.mktmpdir do |temp_cookbooks_path|
+              proxy_cookbook_path = "#{temp_cookbooks_path}/#{cookbook_name}"
+
+              # Make a symlink
+              file_class.symlink other.file_path, proxy_cookbook_path
+
+              # Instantiate a proxy loader using the temporary symlink
+              proxy_loader = Chef::Cookbook::CookbookVersionLoader.new(proxy_cookbook_path, other.parent.chefignore)
+              proxy_loader.load_cookbooks
+
+              cookbook_to_upload = proxy_loader.cookbook_version
+              cookbook_to_upload.identifier = identifier
+              cookbook_to_upload.freeze_version if options[:freeze]
+
+              # Instantiate a new uploader based on the proxy loader
+              uploader = Chef::CookbookUploader.new(cookbook_to_upload, force: options[:force], rest: root.chef_rest, policy_mode: true)
+
+              with_actual_cookbooks_dir(temp_cookbooks_path) do
+                uploader.upload_cookbooks
+              end
+
+              #
+              # When the temporary directory is being deleted on
+              # windows, the contents of the symlink under that
+              # directory is also deleted. So explicitly remove
+              # the symlink without removing the original contents if we
+              # are running on windows
+              #
+              if Chef::Platform.windows?
+                Dir.rmdir proxy_cookbook_path
+              end
+            end
+          end
+
+          def can_have_child?(name, is_dir)
+            is_dir && name.include?('-')
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/chef_server/cookbooks_dir.rb b/lib/chef/chef_fs/file_system/chef_server/cookbooks_dir.rb
index 660cc95734..ce74ac1a5c 100644
--- a/lib/chef/chef_fs/file_system/chef_server/cookbooks_dir.rb
+++ b/lib/chef/chef_fs/file_system/chef_server/cookbooks_dir.rb
@@ -29,15 +29,13 @@ class Chef
   module ChefFS
     module FileSystem
       module ChefServer
-        # cookbooks/
+        #
+        # /cookbooks
+        #
+        # Example children:
         #   apache2/
         #   mysql/
-        # cookbook_artifacts/
-        #   apache2/
-        #     1.0.0/
-        #     1.0.1/
-        #   mysql/
-        #     2.0.5/
+        #
         class CookbooksDir < RestListDir
 
           include Chef::Mixin::FileClass
diff --git a/lib/chef/chef_fs/file_system/chef_server/versioned_cookbook_dir.rb b/lib/chef/chef_fs/file_system/chef_server/versioned_cookbook_dir.rb
index ddc3e7a049..0ffd156577 100644
--- a/lib/chef/chef_fs/file_system/chef_server/versioned_cookbook_dir.rb
+++ b/lib/chef/chef_fs/file_system/chef_server/versioned_cookbook_dir.rb
@@ -16,7 +16,7 @@
 # limitations under the License.
 #
 
-require "chef/chef_fs/file_system/chef_server/versioned_cookbook_dir"
+require "chef/chef_fs/file_system/chef_server/cookbook_dir"
 
 class Chef
   module ChefFS
diff --git a/lib/chef/chef_fs/file_system/chef_server/versioned_cookbooks_dir.rb b/lib/chef/chef_fs/file_system/chef_server/versioned_cookbooks_dir.rb
index c22723d1e2..0b1480cb2d 100644
--- a/lib/chef/chef_fs/file_system/chef_server/versioned_cookbooks_dir.rb
+++ b/lib/chef/chef_fs/file_system/chef_server/versioned_cookbooks_dir.rb
@@ -23,14 +23,21 @@ class Chef
   module ChefFS
     module FileSystem
       module ChefServer
-        # /cookbooks
         #
-        # Its children look like:
+        # /cookbooks or /cookbook_artifacts
+        #
+        # Example children of /cookbooks:
         #
         # - apache2-1.0.0
         # - apache2-1.0.1
         # - mysql-2.0.5
         #
+        # Example children of /cookbook_artifacts:
+        #
+        # - apache2-ab234098245908ddf324a
+        # - apache2-295387a9823745feff239
+        # - mysql-1a2b9e1298734dfe90444
+        #
         class VersionedCookbooksDir < CookbooksDir
 
           def make_child_entry(name)
diff --git a/lib/chef/chef_fs/file_system/repository/chef_repository_file_system_root_dir.rb b/lib/chef/chef_fs/file_system/repository/chef_repository_file_system_root_dir.rb
index 3cb3c914b2..2add4072ea 100644
--- a/lib/chef/chef_fs/file_system/repository/chef_repository_file_system_root_dir.rb
+++ b/lib/chef/chef_fs/file_system/repository/chef_repository_file_system_root_dir.rb
@@ -168,6 +168,8 @@ class Chef
               else
                 dirs = paths.map { |path| ChefRepositoryFileSystemCookbooksDir.new(name, self, path) }
               end
+            when 'cookbook_artifacts'
+              dirs = paths.map { |path| ChefRepositoryFileSystemVersionedCookbooksDir.new(name, self, path) }
             when "data_bags"
               dirs = paths.map { |path| ChefRepositoryFileSystemDataBagsDir.new(name, self, path) }
             when "acls"
diff --git a/spec/integration/knife/download_spec.rb b/spec/integration/knife/download_spec.rb
index faa640f83b..57229b915a 100644
--- a/spec/integration/knife/download_spec.rb
+++ b/spec/integration/knife/download_spec.rb
@@ -1148,6 +1148,7 @@ Created /containers/nodes.json
 Created /containers/policies.json
 Created /containers/roles.json
 Created /containers/sandboxes.json
+Created /cookbook_artifacts
 Created /cookbooks
 Created /data_bags
 Created /environments
@@ -1172,6 +1173,7 @@ EOM
           # acl_for %w(organizations foo groups blah)
           client "x", {}
           cookbook "x", "1.0.0"
+          cookbook_artifact "x", "1x1", { "metadata.rb" => cb_metadata("x", "1.0.0") }
           container "x", {}
           data_bag "x", { "y" => {} }
           environment "x", {}
@@ -1184,7 +1186,7 @@ EOM
           policy_group "x", {
             "policies" => {
               "x" => { "revision_id" => "1.0.0" },
-              "blah" => { "revision_id" => "1.0.0" },
+              "blah" => { "revision_id" => "1.0.0" }
             }
           }
           role "x", {}
@@ -1209,6 +1211,8 @@ EOM
           knife("download /").should_succeed <<EOM
 Created /clients/x.json
 Created /containers/x.json
+Created /cookbook_artifacts/x-1x1
+Created /cookbook_artifacts/x-1x1/metadata.rb
 Created /cookbooks/x
 Created /cookbooks/x/metadata.rb
 Created /data_bags/x
@@ -1233,6 +1237,7 @@ EOM
             file "clients/x.json", { "public_key" => ChefZero::PUBLIC_KEY }
             file "containers/x.json", {}
             file "cookbooks/x/metadata.rb", cb_metadata("x", "1.0.0")
+            file "cookbook_artifacts/x-1x1/metadata.rb", cb_metadata("x", "1.0.0")
             file "data_bags/x/y.json", {}
             file "environments/x.json", {}
             file "groups/x.json", {}
@@ -1257,6 +1262,7 @@ EOM
             file "clients/x.json", { "validator" => true }
             file "containers/x.json", {}
             file "cookbooks/x/metadata.rb", cb_metadata("x", "1.0.1")
+            file "cookbook_artifacts/x-1x1/metadata.rb", cb_metadata("x", "1.0.1")
             file "data_bags/x/y.json", { "a" => "b" }
             file "environments/x.json", { "description" => "foo" }
             file "groups/x.json", { "description" => "foo" }
@@ -1267,7 +1273,7 @@ EOM
             file "policy_groups/x.json", {
               "policies" => {
                 "x" => { "revision_id" => "1.0.1" },
-                "y" => { "revision_id" => "1.0.0" },
+                "y" => { "revision_id" => "1.0.0" }
               }
             }
             file "roles/x.json", { "run_list" => [ "blah" ] }
@@ -1276,6 +1282,7 @@ EOM
           it "knife download updates everything" do
             knife("download /").should_succeed <<EOM
 Updated /clients/x.json
+Updated /cookbook_artifacts/x-1x1/metadata.rb
 Updated /cookbooks/x/metadata.rb
 Updated /data_bags/x/y.json
 Updated /environments/x.json
diff --git a/spec/integration/knife/list_spec.rb b/spec/integration/knife/list_spec.rb
index 32f5bb43f8..85eb959847 100644
--- a/spec/integration/knife/list_spec.rb
+++ b/spec/integration/knife/list_spec.rb
@@ -661,6 +661,7 @@ EOM
 /acls
 /clients
 /containers
+/cookbook_artifacts
 /cookbooks
 /data_bags
 /environments
@@ -681,6 +682,7 @@ EOM
 acls
 clients
 containers
+cookbook_artifacts
 cookbooks
 data_bags
 environments
@@ -753,6 +755,8 @@ policies.json
 roles.json
 sandboxes.json
 
+/cookbook_artifacts:
+
 /cookbooks:
 
 /data_bags:
@@ -783,6 +787,7 @@ EOM
 acls
 clients
 containers
+cookbook_artifacts
 cookbooks
 data_bags
 environments
@@ -855,6 +860,8 @@ policies.json
 roles.json
 sandboxes.json
 
+/cookbook_artifacts:
+
 /cookbooks:
 
 /data_bags:
@@ -886,6 +893,8 @@ EOM
         container "container2", {}
         cookbook "cookbook1", "1.0.0"
         cookbook "cookbook2", "1.0.1", { "recipes" => { "default.rb" => "" } }
+        cookbook_artifact "cookbook_artifact1", "1x1"
+        cookbook_artifact "cookbook_artifact2", "2x2", { "recipes" => { "default.rb" => "" } }
         data_bag "bag1", { "item1" => {}, "item2" => {} }
         data_bag "bag2", { "item1" => {}, "item2" => {} }
         environment "environment1", {}
@@ -970,6 +979,13 @@ EOM
 /containers/policies.json
 /containers/roles.json
 /containers/sandboxes.json
+/cookbook_artifacts/
+/cookbook_artifacts/cookbook_artifact1-1x1/
+/cookbook_artifacts/cookbook_artifact1-1x1/metadata.rb
+/cookbook_artifacts/cookbook_artifact2-2x2/
+/cookbook_artifacts/cookbook_artifact2-2x2/metadata.rb
+/cookbook_artifacts/cookbook_artifact2-2x2/recipes/
+/cookbook_artifacts/cookbook_artifact2-2x2/recipes/default.rb
 /cookbooks/
 /cookbooks/cookbook1/
 /cookbooks/cookbook1/metadata.rb
diff --git a/spec/integration/knife/upload_spec.rb b/spec/integration/knife/upload_spec.rb
index 79dae99acf..6ef40a1d98 100644
--- a/spec/integration/knife/upload_spec.rb
+++ b/spec/integration/knife/upload_spec.rb
@@ -1288,6 +1288,7 @@ EOM
           file "clients/x.json", { "public_key" => ChefZero::PUBLIC_KEY }
           file "containers/x.json", {}
           file "cookbooks/x/metadata.rb", cb_metadata("x", "1.0.0")
+          file "cookbook_artifacts/x-1x1/metadata.rb", cb_metadata("x", "1.0.0")
           file "data_bags/x/y.json", {}
           file "environments/x.json", {}
           file "groups/x.json", {}
@@ -1306,6 +1307,7 @@ EOM
 Updated /acls/groups/blah.json
 Created /clients/x.json
 Created /containers/x.json
+Created /cookbook_artifacts/x-1x1
 Created /cookbooks/x
 Created /data_bags/x
 Created /data_bags/x/y.json
@@ -1322,6 +1324,7 @@ Created /roles/x.json
 EOM
           expect(api.get("association_requests").map { |a| a["username"] }).to eq([ "foo" ])
           expect(api.get("users").map { |a| a["user"]["username"] }).to eq([ "bar" ])
+          knife("diff --name-status --diff-filter=AMT /").should_succeed ""
         end
 
         context "When the chef server has an identical copy of each thing" do
@@ -1333,6 +1336,7 @@ EOM
             # acl_for %w(organizations foo groups blah)
             client "x", {}
             cookbook "x", "1.0.0"
+            cookbook_artifact "x", "1x1", "metadata.rb" => cb_metadata("x", "1.0.0")
             container "x", {}
             data_bag "x", { "y" => {} }
             environment "x", {}
@@ -1345,7 +1349,7 @@ EOM
             policy_group "x", {
               "policies" => {
                 "x" => { "revision_id" => "1.0.0" },
-                "blah" => { "revision_id" => "1.0.0" },
+                "blah" => { "revision_id" => "1.0.0" }
               }
             }
             role "x", {}
@@ -1370,12 +1374,25 @@ EOM
           end
         end
 
+        context "When the chef server has a slightly different copy of the cookbook artifact" do
+          before do
+            cookbook_artifact "x", "1x1", { "recipes" => { "default.rb" => "" } }
+          end
+
+          it "should fail because cookbook_artifacts cannot be updated" do
+            knife("upload /cookbook_artifacts/x-1x1").should_fail <<EOM
+ERROR: /cookbook_artifacts/x-1x1 cannot be updated: cookbook artifacts are immutable once uploaded.
+EOM
+          end
+        end
+
         context "When the chef server has a slightly different copy of each thing (except policy revisions)" do
           before do
             # acl_for %w(organizations foo groups blah)
             client "x", { "validator" => true }
             container "x", {}
             cookbook "x", "1.0.0", { "recipes" => { "default.rb" => "" } }
+            cookbook_artifact "x", "1x1", { "metadata.rb" => cb_metadata("x", "1.0.0") }
             data_bag "x", { "y" => { "a" => "b" } }
             environment "x", { "description" => "foo" }
             group "x", { "groups" => [ "admin" ] }
@@ -1386,7 +1403,7 @@ EOM
             policy_group "x", {
               "policies" => {
                 "x" => { "revision_id" => "1.0.1" },
-                "y" => { "revision_id" => "1.0.0" },
+                "y" => { "revision_id" => "1.0.0" }
               }
             }
             role "x", { "run_list" => [ "blah" ] }