diff options
author | José Valim <jose.valim@dashbit.co> | 2023-04-22 20:27:15 +0200 |
---|---|---|
committer | José Valim <jose.valim@dashbit.co> | 2023-04-22 23:45:44 +0200 |
commit | 12d1b95c03dc1f7212ea7aed7f27ed73751cb076 (patch) | |
tree | f23d9789b17de43dd975404c4c89c5bc882735b9 | |
parent | 9139b4da9919095b615c35ae7572dfe3693dd04a (diff) | |
download | elixir-12d1b95c03dc1f7212ea7aed7f27ed73751cb076.tar.gz |
Automatically recompile dependencies if compile env changes
-rw-r--r-- | lib/elixir/lib/config/provider.ex | 11 | ||||
-rw-r--r-- | lib/mix/lib/mix.ex | 2 | ||||
-rw-r--r-- | lib/mix/lib/mix/app_loader.ex | 127 | ||||
-rw-r--r-- | lib/mix/lib/mix/dep/converger.ex | 2 | ||||
-rw-r--r-- | lib/mix/lib/mix/dep/loader.ex | 21 | ||||
-rw-r--r-- | lib/mix/lib/mix/state.ex | 2 | ||||
-rw-r--r-- | lib/mix/lib/mix/tasks/compile.all.ex | 18 | ||||
-rw-r--r-- | lib/mix/lib/mix/tasks/compile.ex | 2 | ||||
-rw-r--r-- | lib/mix/test/fixtures/deps_status/custom/raw_repo/lib/raw_repo.ex | 2 | ||||
-rw-r--r-- | lib/mix/test/mix/tasks/deps_test.exs | 61 |
10 files changed, 143 insertions, 105 deletions
diff --git a/lib/elixir/lib/config/provider.ex b/lib/elixir/lib/config/provider.ex index c8e27df09..cfd7d16b1 100644 --- a/lib/elixir/lib/config/provider.ex +++ b/lib/elixir/lib/config/provider.ex @@ -274,6 +274,17 @@ defmodule Config.Provider do end @doc false + def valid_compile_env?(compile_env) do + Enum.all?(compile_env, fn {app, [key | path], compile_return} -> + try do + traverse_env(Application.fetch_env(app, key), path) == compile_return + rescue + _ -> false + end + end) + end + + @doc false def validate_compile_env(compile_env, ensure_loaded? \\ true) def validate_compile_env([{app, [key | path], compile_return} | compile_env], ensure_loaded?) do diff --git a/lib/mix/lib/mix.ex b/lib/mix/lib/mix.ex index dba6f114f..ae8c6c44b 100644 --- a/lib/mix/lib/mix.ex +++ b/lib/mix/lib/mix.ex @@ -593,7 +593,7 @@ defmodule Mix do """ def ensure_application!(app) when is_atom(app) do case Mix.State.builtin_apps() do - %{^app => {:ebin, path}} -> + %{^app => path} -> Code.prepend_path(path, cache: true) %{} -> diff --git a/lib/mix/lib/mix/app_loader.ex b/lib/mix/lib/mix/app_loader.ex index 1c98473f1..fdb0fb61d 100644 --- a/lib/mix/lib/mix/app_loader.ex +++ b/lib/mix/lib/mix/app_loader.ex @@ -52,51 +52,43 @@ defmodule Mix.AppLoader do end @doc """ - Loads the given application from `ebin_path`. - - Returns either `{:ok, children}` or `{:error, message}`. + Loads the given app from path in an optimized format and returns its contents. """ - def load_app(app, ebin_path, validate_compile_env?) do - if Application.spec(app, :vsn) do - {:ok, children(app)} - else - with true <- ebin_path != nil, - {:ok, bin} <- File.read(app_join(ebin_path, app, ~c".app")), - {:ok, {:application, _, properties} = application_data} <- consult_app_file(bin), - :ok <- :application.load(application_data) do - with [_ | _] = compile_env <- validate_compile_env? && properties[:compile_env], - {:error, message} <- Config.Provider.validate_compile_env(compile_env, false) do - {:error, message} + def load_app(app, app_path) do + case File.read(app_path) do + {:ok, bin} -> + with {:ok, tokens, _} <- :erl_scan.string(String.to_charlist(bin)), + {:ok, {:application, ^app, properties} = app_data} <- :erl_parse.parse_term(tokens), + :ok <- ensure_loaded(app_data) do + {:ok, properties} else - _ -> {:ok, children(app)} + _ -> :invalid end - else - # Optional applications won't be available - _ -> {:ok, []} - end + + {:error, _} -> + :missing + end + end + + defp ensure_loaded(app_data) do + case :application.load(app_data) do + :ok -> :ok + {:error, {:already_loaded, _}} -> :ok + {:error, error} -> {:error, error} end end @doc """ Loads the given applications. """ - def load_apps(apps, deps, config, validate_compile_env?, acc, fun) do + def load_apps(apps, deps, config, acc, fun) do lib_path = to_charlist(Path.join(Mix.Project.build_path(config), "lib")) deps_children = for dep <- deps, into: %{}, do: {dep.app, Enum.map(dep.deps, & &1.app)} - deps_paths = for dep <- deps, into: %{}, do: {dep.app, {:lib, lib_path}} builtin_paths = Mix.State.builtin_apps() - paths = Map.merge(builtin_paths, deps_paths) - - ref = make_ref() - parent = self() - opts = [ordered: false, timeout: :infinity] - stream = - (extra_apps(config) ++ apps) - |> stream_apps(deps_children, paths, ref) - |> Task.async_stream(&load_stream_app(&1, ref, parent, validate_compile_env?), opts) - - Enum.reduce(stream, acc, fn {:ok, res}, acc -> fun.(res, acc) end) + (extra_apps(config) ++ apps) + |> traverse_apps(%{}, deps_children, builtin_paths, lib_path) + |> Enum.reduce(acc, fun) end defp extra_apps(config) do @@ -106,60 +98,47 @@ defmodule Mix.AppLoader do end end - defp load_stream_app({app, app_path}, ref, parent, validate_compile_env?) do - ebin_path = app_path_to_ebin_path(app, app_path) - send(parent, {ref, app, load_app(app, ebin_path, validate_compile_env?)}) - {app, ebin_path} - end - - defp stream_apps(initial, deps_children, paths, ref) do - Stream.unfold({initial, %{}, %{}, deps_children, paths, ref}, &stream_app/1) - end - # We already processed this app, skip it. - defp stream_app({[app | apps], seen, done, deps_children, paths, ref}) + defp traverse_apps([app | apps], seen, deps_children, builtin_paths, lib_path) when is_map_key(seen, app) do - stream_app({apps, seen, done, deps_children, paths, ref}) + traverse_apps(apps, seen, deps_children, builtin_paths, lib_path) end # We haven't processed this app, emit it. - defp stream_app({[app | apps], seen, done, deps_children, paths, ref}) do - {{app, paths[app]}, {apps, Map.put(seen, app, true), done, deps_children, paths, ref}} - end + defp traverse_apps([app | apps], seen, deps_children, builtin_paths, lib_path) do + {ebin_path, dep_children} = + case deps_children do + %{^app => dep_children} -> {app_join(lib_path, app, ~c"/ebin"), dep_children} + _ -> {builtin_paths[app], []} + end - # We have processed all apps and all seen have been done. - defp stream_app({[], seen, done, _deps_children, _paths, _ref}) - when map_size(seen) == map_size(done) do - nil - end + app_children = + if Application.spec(app, :vsn) do + app_children(app) + else + with true <- ebin_path != nil, + {:ok, _} <- load_app(app, app_join(ebin_path, app, ~c".app")) do + app_children(app) + else + # Optional applications won't be available + _ -> [] + end + end - # We have processed all apps but there is work being done. - defp stream_app({[], seen, done, deps_children, paths, ref}) do - receive do - {^ref, app, {:ok, children}} -> - dep_children = Map.get(deps_children, app, []) - children = (dep_children -- children) ++ children - stream_app({children, seen, Map.put(done, app, true), deps_children, paths, ref}) + children = (dep_children -- app_children) ++ app_children + seen = Map.put(seen, app, true) + apps = children ++ apps + [{app, ebin_path} | traverse_apps(apps, seen, deps_children, builtin_paths, lib_path)] + end - {^ref, _app, {:error, message}} -> - Mix.raise(message) - end + # We have processed all apps. + defp traverse_apps([], _seen, _deps_children, _builtin_paths, _lib_path) do + [] end - defp children(app) do + defp app_children(app) do Application.spec(app, :applications) ++ Application.spec(app, :included_applications) end - defp app_path_to_ebin_path(app, {:lib, lib_path}), do: app_join(lib_path, app, ~c"/ebin") - defp app_path_to_ebin_path(_app, {:ebin, ebin_path}), do: ebin_path - defp app_path_to_ebin_path(_app, nil), do: nil - defp app_join(path, app, suffix), do: path ++ ~c"/" ++ Atom.to_charlist(app) ++ suffix - - defp consult_app_file(bin) do - # The path could be located in an .ez archive, so we use the prim loader. - with {:ok, tokens, _} <- :erl_scan.string(String.to_charlist(bin)) do - :erl_parse.parse_term(tokens) - end - end end diff --git a/lib/mix/lib/mix/dep/converger.ex b/lib/mix/lib/mix/dep/converger.ex index 7ae5f4928..87adb3206 100644 --- a/lib/mix/lib/mix/dep/converger.ex +++ b/lib/mix/lib/mix/dep/converger.ex @@ -108,7 +108,7 @@ defmodule Mix.Dep.Converger do use_remote? = !!remote and Enum.any?(deps, &remote.remote?/1) if not diverged? and use_remote? do - # Make sure there are no cycles before calling remote converge + # Make sure there are no cycles before calling the remote converger topological_sort(deps) # If there is a lock, it means we are doing a get/update diff --git a/lib/mix/lib/mix/dep/loader.ex b/lib/mix/lib/mix/dep/loader.ex index 11e707011..31aa9d503 100644 --- a/lib/mix/lib/mix/dep/loader.ex +++ b/lib/mix/lib/mix/dep/loader.ex @@ -411,14 +411,14 @@ defmodule Mix.Dep.Loader do end defp app_status(app_path, app, req) do - case :file.consult(app_path) do - {:ok, [{:application, ^app, config}]} -> - case List.keyfind(config, :vsn, 0) do + case Mix.AppLoader.load_app(app, app_path) do + {:ok, properties} -> + case List.keyfind(properties, :vsn, 0) do {:vsn, actual} when is_list(actual) -> actual = IO.iodata_to_binary(actual) case vsn_match(req, actual, app) do - {:ok, true} -> {:ok, actual} + {:ok, true} -> compile_env_status(actual, properties) {:ok, false} -> {:nomatchvsn, actual} {:error, error} -> {error, actual} end @@ -430,14 +430,23 @@ defmodule Mix.Dep.Loader do {:invalidvsn, nil} end - {:ok, _} -> + :invalid -> {:invalidapp, app_path} - {:error, _} -> + :missing -> case Path.wildcard(Path.join(Path.dirname(app_path), "*.app")) do [other_app_path] -> {:noappfile, {app_path, other_app_path}} _ -> {:noappfile, {app_path, nil}} end end end + + defp compile_env_status(vsn, properties) do + with [_ | _] = compile_env <- properties[:compile_env], + false <- Config.Provider.valid_compile_env?(compile_env) do + :compile + else + _ -> {:ok, vsn} + end + end end diff --git a/lib/mix/lib/mix/state.ex b/lib/mix/lib/mix/state.ex index f97a50148..19a18f987 100644 --- a/lib/mix/lib/mix/state.ex +++ b/lib/mix/lib/mix/state.ex @@ -107,7 +107,7 @@ defmodule Mix.State do builtin_apps = for path <- builtin_apps, app = app_from_code_path(path), - do: {app, {:ebin, path}}, + do: {app, path}, into: %{} {:reply, builtin_apps, %{state | builtin_apps: builtin_apps}} diff --git a/lib/mix/lib/mix/tasks/compile.all.ex b/lib/mix/lib/mix/tasks/compile.all.ex index ddc83bb19..d48e17dc3 100644 --- a/lib/mix/lib/mix/tasks/compile.all.ex +++ b/lib/mix/lib/mix/tasks/compile.all.ex @@ -27,14 +27,12 @@ defmodule Mix.Tasks.Compile.All do # from archives will be removed from the code path. deps = Mix.Dep.cached() apps = project_apps(config) - validate_compile_env? = "--no-validate-compile-env" not in args {loaded_paths, loaded_modules} = - Mix.AppLoader.load_apps(apps, deps, config, validate_compile_env?, {[], []}, fn - {app, path}, {paths, mods} -> - paths = if path, do: [path | paths], else: paths - mods = if app_cache, do: [{app, Application.spec(app, :modules)} | mods], else: mods - {paths, mods} + Mix.AppLoader.load_apps(apps, deps, config, {[], []}, fn {app, path}, {paths, mods} -> + paths = if path, do: [path | paths], else: paths + mods = if app_cache, do: [{app, Application.spec(app, :modules)} | mods], else: mods + {paths, mods} end) # We compute the diff as that will be more efficient @@ -73,8 +71,12 @@ defmodule Mix.Tasks.Compile.All do _ = Code.prepend_path(compile_path) unless "--no-app-loading" in args do - with {:error, message} <- - Mix.AppLoader.load_app(config[:app], compile_path, validate_compile_env?) do + app = config[:app] + + with {:ok, properties} <- Mix.AppLoader.load_app(app, "#{compile_path}/#{app}.app"), + false <- "--no-validate-compile-env" in args, + [_ | _] = compile_env <- properties[:compile_env], + {:error, message} <- Config.Provider.validate_compile_env(compile_env, false) do Mix.raise(message) end end diff --git a/lib/mix/lib/mix/tasks/compile.ex b/lib/mix/lib/mix/tasks/compile.ex index b884f07d7..758bed682 100644 --- a/lib/mix/lib/mix/tasks/compile.ex +++ b/lib/mix/lib/mix/tasks/compile.ex @@ -146,7 +146,7 @@ defmodule Mix.Tasks.Compile do loaded_paths = Mix.Project.apps_paths(config) |> Map.keys() - |> Mix.AppLoader.load_apps(Mix.Dep.cached(), config, false, [], fn + |> Mix.AppLoader.load_apps(Mix.Dep.cached(), config, [], fn {_app, path}, acc -> if path, do: [path | acc], else: acc end) diff --git a/lib/mix/test/fixtures/deps_status/custom/raw_repo/lib/raw_repo.ex b/lib/mix/test/fixtures/deps_status/custom/raw_repo/lib/raw_repo.ex index 806471ca9..b893d48bc 100644 --- a/lib/mix/test/fixtures/deps_status/custom/raw_repo/lib/raw_repo.ex +++ b/lib/mix/test/fixtures/deps_status/custom/raw_repo/lib/raw_repo.ex @@ -1,3 +1,5 @@ +Application.compile_env(:raw_repo, :compile_env) + defmodule RawRepo do def hello do "world" diff --git a/lib/mix/test/mix/tasks/deps_test.exs b/lib/mix/test/mix/tasks/deps_test.exs index a16044409..200a978d9 100644 --- a/lib/mix/test/mix/tasks/deps_test.exs +++ b/lib/mix/test/mix/tasks/deps_test.exs @@ -57,6 +57,18 @@ defmodule Mix.Tasks.DepsTest do end end + defmodule RawRepoDep do + def project do + [ + app: :raw_sample, + version: "0.1.0", + deps: [ + {:raw_repo, "0.1.0", path: "custom/raw_repo"} + ] + ] + end + end + ## deps test "prints list of dependencies and their status" do @@ -414,18 +426,6 @@ defmodule Mix.Tasks.DepsTest do ## Deps environment - defmodule DepsEnvApp do - def project do - [ - app: :raw_sample, - version: "0.1.0", - deps: [ - {:raw_repo, "0.1.0", path: "custom/raw_repo"} - ] - ] - end - end - defmodule CustomDepsEnvApp do def project do [ @@ -440,7 +440,7 @@ defmodule Mix.Tasks.DepsTest do test "sets deps env to prod by default" do in_fixture("deps_status", fn -> - Mix.Project.push(DepsEnvApp) + Mix.Project.push(RawRepoDep) Mix.Tasks.Deps.Update.run(["--all"]) assert_received {:mix_shell, :info, [":raw_repo env is prod"]} @@ -775,6 +775,41 @@ defmodule Mix.Tasks.DepsTest do end) end + test "checks if compile env changed" do + in_fixture("deps_status", fn -> + Mix.Project.push(RawRepoDep) + Mix.Tasks.Deps.Loadpaths.run([]) + assert_receive {:mix_shell, :info, ["Generated raw_repo app"]} + assert Application.spec(:raw_repo, :vsn) + + File.mkdir_p!("config") + + File.write!("config/config.exs", """ + import Config + config :raw_repo, :compile_env, :new_value + """) + + Application.unload(:raw_repo) + Mix.ProjectStack.pop() + Mix.Task.clear() + Mix.Project.push(RawRepoDep) + purge([RawRepo]) + Mix.Tasks.Loadconfig.load_compile("config/config.exs") + + Mix.Tasks.Deps.run([]) + + assert_receive {:mix_shell, :info, + [" the dependency build is outdated, please run \"mix deps.compile\""]} + + Mix.Tasks.Deps.Loadpaths.run([]) + + assert_receive {:mix_shell, :info, ["Generated raw_repo app"]} + assert Application.spec(:raw_repo, :vsn) + end) + after + Application.delete_env(:raw_repo, :compile_env, persistent: true) + end + defmodule NonCompilingDeps do def project do [ |