Robustify dls2 shutdown procedure

modules/application/include/dls2/application/layer.hpp

@@ -57,6 +57,12 @@ namespace dls
 		template<class Map>
 		void checkAppData(const Map& app_data);
 
+		template<class Map>
+		void checkAppData(const Map& app_data, std::mutex& mutex);
+
+		template<class Map>
+		void checkAppDataImpl(const Map& app_data, std::mutex* mutex);
+
 		// BEGIN critical section
 			mutable std::mutex components_mutex;
 			std::map<std::string, pComponent_t> components;

modules/application/include/dls2/application/layer.tpp

@@ -2,17 +2,33 @@
 #define LAYER_TPP_H7JRIVPM
 
 #include "dls2/application/layer.hpp"
+#include <vector>
 
 namespace dls
 {
 
 template<class Map>
 void Layer::checkAppData(const Map& app_data)
+{
+	checkAppDataImpl(app_data, nullptr);
+}
+
+template<class Map>
+void Layer::checkAppData(const Map& app_data, std::mutex& mutex)
+{
+	checkAppDataImpl(app_data, &mutex);
+}
+
+template<class Map>
+void Layer::checkAppDataImpl(const Map& app_data, std::mutex* mutex)
 {
 	using Ptr = typename Map::mapped_type;
     using Data = typename Ptr::element_type;
     static_assert(std::is_base_of_v<AppData, Data>, "must store AppData-derived");
 
+	std::vector<std::string> stopped;
+	std::unique_lock<std::mutex> lock;
+	if (mutex) lock = std::unique_lock<std::mutex>(*mutex);
 	for(const auto& [key, data] : app_data)
 	{
 		if(!data || !data->proc)
@@ -24,14 +40,14 @@ void Layer::checkAppData(const Map& app_data)
 		{
 			if (this->safety_layer_config_->enable_process_died)
 			{
-				this->robust_event_notifier.notify(
-					EventID::PROCESS_DIED,
-					EventSeverity::ERROR,
-					this->getID() + ": " + key + " is not running"
-				);
+				stopped.push_back(key);
 			}
 		}
 	}
+	if (lock.owns_lock()) lock.unlock();
+	for (const auto& key : stopped)
+		this->robust_event_notifier.notify(EventID::PROCESS_DIED, EventSeverity::ERROR,
+			this->getID() + ": " + key + " is not running");
 }
 
 } // end namespace dls

modules/child_process/src/child_process_launcher.cpp.in

@@ -23,27 +23,14 @@
 // #include "dls2/service/service_base.hpp"
 #include "dls2/log/event_logger.hpp"
 #include "dls2/application/layer.hpp"
+#include "dls2/util/shutdown_signal.hpp"
 
 #include "robotlib/robot_factory.hpp"
 
 // =============================================================================
 // Using Declarations
 // =============================================================================
 using namespace dls;
-std::string process_name("");
-CommandManager stopper("process_launcher");
-void shutdown(int)
-{
-    bool stop_wait = false;
-    if (process_name.compare("")==0)
-        exit(EXIT_FAILURE);
-    else if(stopper.waitCommand(process_name, "shutdown", stop_wait))
-        stopper.callCommand("shutdown", {}, process_name);
-    else{
-        std::cerr << process_name <<"::shutdown not found. Process is brutally killed.";
-        exit(EXIT_FAILURE);
-    }
-}
 
 // =============================================================================
 // Globals
@@ -94,14 +81,11 @@ void change_process_name(char **argv, const std::string &name)
 // =============================================================================
 int main(int argc, char **argv)
 {
-    // When the user presses CTRL+C, make sure that all layers are shutdown
-    signal(
-        SIGINT,
-        shutdown);
+    std::shared_ptr<App> pComponent;
+    utils::ShutdownSignal shutdown_signal;
     // give time to the logstrem connect to the logger layer
     Args args = parse_args(argc, argv);
 
-    std::shared_ptr<App> pComponent;
 
     // Load the component
     try
@@ -206,7 +190,15 @@ int main(int argc, char **argv)
         exit(EXIT_FAILURE);
     }
 
-    process_name = pComponent->getID();
+    shutdown_signal.start([pComponent, is_console = args.component_type == "layer" && args.lib_name == "console"] {
+        // Initialization has no quit transition. Retain an early signal until idle.
+        const auto state = pComponent->sm.getStateName();
+        if (state == "initialization") return false;
+        // Ctrl+C in a separate console must also stop the connected framework.
+        if (is_console) pComponent->command_manager.callCommand("shutdown", {"all"});
+        if (state != "quit") pComponent->stop();
+        return true;
+    });
 
     change_process_name(argv, args.component_name.c_str());
 

modules/command/src/command_manager.cpp

@@ -6,6 +6,11 @@
 
 using namespace dls;
 
+namespace
+{
+	constexpr int wait_poll_period_ms = 10;
+}
+
 CommandManager::CommandManager(std::string owner_)
 	: commands()
 	, owner(owner_)
@@ -126,7 +131,7 @@ std::multimap<std::string, std::string> CommandManager::getCommandsList()
 	if(command_publisher_listener == nullptr)
 		return {};
 	// Get matched datareaders instances
-	auto matched_datareaders_instances = command_publisher_listener->matched_datareaders_instances;
+	auto matched_datareaders_instances = command_publisher_listener->get_matched_datareaders_instances();
 	// Find the domain participant name associated to each matched data reader, and save the name (corresponding to the command name)
 	std::multimap<std::string, std::string> cmds;
 	for(auto datareader_instance : matched_datareaders_instances)
@@ -302,7 +307,7 @@ bool CommandManager::waitCommand(const std::string& owner, const std::string& na
 						return false;
 					}
 					return true;
-				}), timeout_ms, 2, stop_wait)){
+				}), timeout_ms, wait_poll_period_ms, stop_wait)){
 			if(!stop_wait)
 				std::cerr << "Command " << owner << "::" << name<<" not found" << std::endl;
 			return false;
@@ -316,7 +321,7 @@ bool CommandManager::waitCommand(const std::string& owner, const std::string& na
 						return false;
 					}
 					return true;
-				}), timeout_ms, 2, stop_wait)){
+				}), timeout_ms, wait_poll_period_ms, stop_wait)){
 			if(!stop_wait.load())
 				std::cerr << "Command " << owner << "::" << name<<" not found" << std::endl;
 			return false;

modules/core_framework/include/dls2/core_framework/console_layer.hpp

@@ -6,6 +6,7 @@
 #include <readline/readline.h>
 #include <readline/history.h>
 #include <filesystem>
+#include <atomic>
 
 namespace dls
 {
@@ -53,11 +54,13 @@ namespace dls
 
 		// needed to unblock the console from the readline
 		void stop() override;
+		bool shutdownRequested() const { return shutdown_requested_.load(); }
 
 	private:
+		std::atomic_bool shutdown_requested_{false};
 		// Map with "load[Layer]" commands and associated installation folders
 		const std::map<std::string, std::string> load_layers_paths_;
 	};
 } // namespace dls
 
-#endif
+#endif

modules/core_framework/include/dls2/core_framework/control_layer.hpp

@@ -103,6 +103,11 @@ class ControlLayer : public Layer
 		std::mutex motion_mutex;
 	// END critical section
 
+	// Serialize operations without holding map locks across DDS or shutdown waits.
+	std::mutex controllers_operations_mutex;
+	std::mutex motion_operations_mutex;
+	std::atomic_bool closing_{false};
+
 	std::shared_ptr<dls::DDSParticipant> ddsSignalLink;
 
 	/// Default controller spline-in

modules/core_framework/src/console_layer.cpp.in

@@ -225,11 +225,16 @@ namespace dls
 				rl_reset_line_state();
 			}
 		}
-		int event(void){return 0;}
+		int event(void)
+		{
+			// Readline state is accessed only by the thread running readline.
+			if (console_layer && console_layer->shutdownRequested()) rl_done = 1;
+			return 0;
+		}
 	} // namespace readline_completion
 
 	// dls::ConsoleLayer implementation
-	ConsoleLayer::ConsoleLayer(std::string ID) : Layer(ID, 50),
+	ConsoleLayer::ConsoleLayer(std::string ID) : Layer(ID, 3000),
 												 load_layers_paths_{{"loadController", "${DLS_INSTALL_CONTROLLER_DIR}"},
 																	{"loadGenerator", "${DLS_INSTALL_MOTION_GENERATOR_DIR}"},
 																	{"loadEstimator", "${DLS_INSTALL_ESTIMATOR_DIR}"},
@@ -242,8 +247,8 @@ namespace dls
 	{
 		// Assign the dls::readline_completion::ConsoleLayer pointer to the dls::ConsoleLayer object
 		readline_completion::console_layer = this;
-		// Make readline reading rl_done variable, set when shutting down the control layer in the stop() function.
-		// When rl_done variable is set to a value !=0, readline return immediately. So the console layer will not wait for a character before exiting
+		// Keep the launcher's signal handler and unblock input on a shutdown request.
+		rl_catch_signals = 0;
 		rl_event_hook = readline_completion::event;
 
 		command_manager.addCommand<>
@@ -286,10 +291,7 @@ namespace dls
 
 	ConsoleLayer::~ConsoleLayer()
 	{
-		// Free the ConsoleLayer pointer ("dls::readline_completion" namespace) allocated memory
-		delete readline_completion::console_layer;
-
-		// Make the dangling pointer point to "null"
+		// This is a non-owning pointer to this object.
 		readline_completion::console_layer = nullptr;
 	}
 
@@ -352,12 +354,11 @@ namespace dls
 		std::cout << std::endl;
 	}
 
-	void ConsoleLayer::close(){}
+	void ConsoleLayer::close(){ shutdown_requested_.store(true); }
 
 	void ConsoleLayer::stop()
 	{
-		// When rl_done variable is set to a value !=0, readline return immediately. So the console layer will not wait for a character before exiting
-		rl_done = 1;
+		shutdown_requested_.store(true);
 		sm.raiseEvent(sm.quit_request);
 	}
 
@@ -368,11 +369,16 @@ namespace dls
 	}
 
 	void ConsoleLayer::monitor(){
+		if (shutdownRequested()) return;
 		rl_attempted_completion_function = readline_completion::command_completion;
 		rl_completion_display_matches_hook = readline_completion::display_matches;
 
 		// The line "readline" returned is allocated with "malloc". It is freed (manually) when we are done with it
 		char *line = readline(std::string("> ").c_str());
+		if (shutdownRequested()) {
+			free(line);
+			return;
+		}
 
 		if (line != nullptr)
 		{
@@ -440,4 +446,4 @@ namespace dls
 			free(line);
 		}
 	}
-} // namespace dls
+} // namespace dls