# Copyright (C) 2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import logging import lorrycontroller class RemoveGhostJobs(lorrycontroller.LorryControllerRoute): http_method = 'POST' path = '/1.0/remove-ghost-jobs' def run(self, **kwargs): logging.info('%s %s called', self.http_method, self.path) ghost_timeout = self.app_settings['ghost-timeout'] ghosts = [] with self.open_statedb() as statedb: for job_id in statedb.get_running_jobs(): if self.is_ghost_job(statedb, job_id, ghost_timeout): self.exorcise_ghost_job(statedb, job_id) ghosts.append(statedb.get_job_info(job_id)) return { 'killed-ghost-jobs': ghosts, } def is_ghost_job(self, statedb, job_id, ghost_timeout): updated = statedb.get_job_updated(job_id) return self.now(statedb) - updated >= ghost_timeout def now(self, statedb): return statedb.get_current_time() def exorcise_ghost_job(self, statedb, job_id): logging.info('Job %s is a ghost job', job_id) self.mark_job_to_be_killed_in_case_minion_appears(statedb, job_id) self.mark_job_as_terminated(statedb, job_id) def mark_job_to_be_killed_in_case_minion_appears(self, statedb, job_id): statedb.set_kill_job(job_id, True) def mark_job_as_terminated(self, statedb, job_id): statedb.append_to_job_output( job_id, '\nTERMINATED DUE TO GHOST TIMEOUT\n') statedb.set_job_exit(job_id, 127, self.now(statedb), -1) job_info = statedb.get_job_info(job_id) statedb.set_running_job(job_info['path'], None)