From 313ef4bb79f602eecd363b242d23f42689f43711 Mon Sep 17 00:00:00 2001 From: Jay Doane Date: Mon, 14 Dec 2020 18:54:02 -0800 Subject: Support pluggable custodian monitor Enable build time configurable monitor for custodian and remove custom sensu events. --- src/custodian/rebar.config.script | 35 ++++++++++++++++++++ src/custodian/src/custodian.app.src | 29 ----------------- src/custodian/src/custodian.app.src.script | 48 ++++++++++++++++++++++++++++ src/custodian/src/custodian_db_checker.erl | 14 ++------ src/custodian/src/custodian_monitor.erl | 28 ++++++++++++++++ src/custodian/src/custodian_noop_monitor.erl | 35 ++++++++++++++++++++ src/custodian/src/custodian_server.erl | 28 +++++++--------- 7 files changed, 160 insertions(+), 57 deletions(-) create mode 100644 src/custodian/rebar.config.script delete mode 100644 src/custodian/src/custodian.app.src create mode 100644 src/custodian/src/custodian.app.src.script create mode 100644 src/custodian/src/custodian_monitor.erl create mode 100644 src/custodian/src/custodian_noop_monitor.erl diff --git a/src/custodian/rebar.config.script b/src/custodian/rebar.config.script new file mode 100644 index 000000000..f32db974c --- /dev/null +++ b/src/custodian/rebar.config.script @@ -0,0 +1,35 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + + +CouchConfig = case filelib:is_file(os:getenv("COUCHDB_CONFIG")) of + true -> + {ok, Result} = file:consult(os:getenv("COUCHDB_CONFIG")), + Result; + false -> + [] +end, + +CustodianMonitor = case lists:keyfind(custodian_monitor, 1, CouchConfig) of + {custodian_monitor, Module} when Module /= "" -> + list_to_atom(Module); + _ -> + custodian_noop_monitor +end, + +CurrentOpts = case lists:keyfind(erl_opts, 1, CONFIG) of + {erl_opts, Opts} -> Opts; + false -> [] +end, + +CustodianOpts = {d, 'CUSTODIAN_MONITOR', CustodianMonitor}, +lists:keystore(erl_opts, 1, CONFIG, {erl_opts, [CustodianOpts | CurrentOpts]}). diff --git a/src/custodian/src/custodian.app.src b/src/custodian/src/custodian.app.src deleted file mode 100644 index b93b21ebb..000000000 --- a/src/custodian/src/custodian.app.src +++ /dev/null @@ -1,29 +0,0 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - -{application, custodian, - [ - {description, "in your cluster, looking after your stuff"}, - {vsn, git}, - {registered, []}, - {applications, [ - kernel, - stdlib, - couch_log, - config, - couch_event, - couch, - mem3 - ]}, - {mod, { custodian_app, []}}, - {env, []} - ]}. diff --git a/src/custodian/src/custodian.app.src.script b/src/custodian/src/custodian.app.src.script new file mode 100644 index 000000000..551b9c2c3 --- /dev/null +++ b/src/custodian/src/custodian.app.src.script @@ -0,0 +1,48 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +CouchConfig = case filelib:is_file(os:getenv("COUCHDB_CONFIG")) of + true -> + {ok, Result} = file:consult(os:getenv("COUCHDB_CONFIG")), + Result; + false -> + [] +end. + +CustodianMonitorApp = case lists:keyfind(custodian_monitor_app, 1, CouchConfig) of + {custodian_monitor_app, AppName} when AppName /= "" -> + [list_to_atom(AppName)]; + _ -> + [] +end. + +BaseApplications = [ + kernel, + stdlib, + couch_log, + config, + couch_event, + couch, + mem3 +]. + +Applications = CustodianMonitorApp ++ BaseApplications. + +{application, custodian, + [ + {description, "in your cluster, looking after your stuff"}, + {vsn, git}, + {registered, []}, + {applications, Applications}, + {mod, { custodian_app, []}}, + {env, []} + ]}. diff --git a/src/custodian/src/custodian_db_checker.erl b/src/custodian/src/custodian_db_checker.erl index 8308c8ecb..10502dd76 100644 --- a/src/custodian/src/custodian_db_checker.erl +++ b/src/custodian/src/custodian_db_checker.erl @@ -149,17 +149,9 @@ get_stats_db() -> send_missing_db_alert(DbName) -> couch_log:notice("Missing system database ~s", [DbName]), - Command = [ - "send-sensu-event --standalone --critical", - " --output=\"Missing system database ", - binary_to_list(DbName), - "\" --handler=default custodian-missing-db-check"], - os:cmd(lists:concat(Command)). + ?CUSTODIAN_MONITOR:send_missing_db_alert(DbName). + clear_missing_dbs_alert() -> couch_log:notice("All system databases exist.", []), - Command = [ - "send-sensu-event --standalone --ok", - " --output=\"All system databases exist\"", - " --handler=default custodian-missing-db-check"], - os:cmd(lists:concat(Command)). + ?CUSTODIAN_MONITOR:clear_missing_dbs_alert(). diff --git a/src/custodian/src/custodian_monitor.erl b/src/custodian/src/custodian_monitor.erl new file mode 100644 index 000000000..3cca046ed --- /dev/null +++ b/src/custodian/src/custodian_monitor.erl @@ -0,0 +1,28 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(custodian_monitor). + + +% N.B. that callback return values are ignored + +-callback send_missing_db_alert(DbName :: binary()) -> + Ignored :: any(). + + +-callback clear_missing_dbs_alert() -> + Ignored :: any(). + + +-callback send_event( + Name :: string(), Count :: non_neg_integer(), Description :: string()) -> + Ignored :: any(). diff --git a/src/custodian/src/custodian_noop_monitor.erl b/src/custodian/src/custodian_noop_monitor.erl new file mode 100644 index 000000000..5c793aeca --- /dev/null +++ b/src/custodian/src/custodian_noop_monitor.erl @@ -0,0 +1,35 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(custodian_noop_monitor). + + +-behaviour(custodian_monitor). + + +-export([ + send_missing_db_alert/1, + clear_missing_dbs_alert/0, + send_event/3 +]). + + +send_missing_db_alert(_DbName) -> + false. + + +clear_missing_dbs_alert() -> + false. + + +send_event(_Name, _Count, _Description) -> + false. diff --git a/src/custodian/src/custodian_server.erl b/src/custodian/src/custodian_server.erl index 322cc3264..0a21eed23 100644 --- a/src/custodian/src/custodian_server.erl +++ b/src/custodian/src/custodian_server.erl @@ -143,28 +143,22 @@ handle_db_event(_DbName, _Event, _St) -> {ok, nil}. check_shards() -> - [send_sensu_event(Item) || Item <- custodian:summary()]. + [send_event(Item) || Item <- custodian:summary()]. -send_sensu_event({_, Count} = Item) -> - Level = case Count of + +send_event({_, Count} = Item) -> + Description = describe(Item), + Name = check_name(Item), + case Count of 0 -> - "--ok"; + ok; 1 -> - couch_log:critical("~s", [describe(Item)]), - "--critical"; + couch_log:critical("~s", [Description]); _ -> - couch_log:warning("~s", [describe(Item)]), - "--warning" + couch_log:warning("~s", [Description]) end, - Cmd = lists:concat([ - "send-sensu-event --standalone ", - Level, - " --output=\"", - describe(Item), - "\" ", - check_name(Item) - ]), - os:cmd(Cmd). + ?CUSTODIAN_MONITOR:send_event(Name, Count, Description). + describe({{safe, N}, Count}) -> lists:concat([Count, " ", shards(Count), " in cluster with only ", N, -- cgit v1.2.1