Add --autorestart option for automatically restarting crashed components.

infrastructure/control/osimctrl/README.md

@@ -14,10 +14,6 @@ user close to what is really happening.  This is to facilitate debugging and the
 ability to deal with the many possible failure conditions.  This might change in
 the future.
 
-As part of this policy, there is currently no way to stop a component via the
-command line.  One has to attach to the screen and quit it via the OpenSimulator
-console.
-
 # Installation #
 
 1. Copy src/* to a location of your choice
@@ -38,12 +34,54 @@ you could
 
 to run more than one simulator on the same machine.
 
+# Use #
+
+simctrl.py controls a simulator instance whilst robustctrl.py can control a
+Robust grid service instance.
+
+If the OpenSimulator installation is in standalone mode then it can be
+controlled purely with the simctrl.py script.
+
+Both these scripts provide a number of commands to control their components.
+These are
+
+start   - start an instance of the component.
+stop    - stop the component if it is running.
+restart - restart the component if it is running.
+attach  - attach to the current screen instance of the component if it is running.
+status  - get the current status of the component.
+
+There are also a few optional switches
+
+-n, --noattach
+
+Do not automatically attach to the screen component once it has started.
+
+-a, --autorestart
+
+If a component fails, then automatically restart it.  If you use this option
+then you can only stop the component with the stop command instead of manually
+attaching to the screen and shutting it down (since the surrounding loop will
+simply start it again).  Alternatively, you can send a SIGTERM to the bash
+script which acts as the simple loop.  This will be a child process from the
+screen instance).
+
+-h. --help
+
+Display usage text
+
+It provides start, stop, restart, attach and status commands, along with
+--noattach and --autorestart options
+
 # Files #
 
 ## robustctrl.py, simctrl.py ##
 These files control a robust instance and a simulator respectively.
 
-Available commands are attach, start and status.  See --help for more details.
+# Bugs #
+
+Because stop currently involves stuff text onto the screen buffer, this will
+fail if there is something already there.
 
 # References #
 [1] http://argparse.googlecode.com/hg/argparse.py

infrastructure/control/osimctrl/src/osimctrl/osimctrl.py

@@ -1,6 +1,7 @@
 import argparse
 import os.path
 import re
+import signal
 import string
 import subprocess
 import sys
@@ -68,7 +69,7 @@ class OSimCtrl:
     elif command == "start":
       self.startComponent(opts)
     elif command == "stop":
-      self.stopComponent()
+      self.stopComponent(opts)
     elif command == "restart":
       self.restartComponent(opts)
     else:
@@ -82,7 +83,7 @@ class OSimCtrl:
       print "Did not find screen named %s for attach" % self._screenName
       return False
     else:
-      print "Found screen %s" % screen.group(1) 
+      print "Found screen %s" % screen 
       execCmd("%s -x %s" % (self._screenPath, self._screenName))
       return True
     
@@ -95,7 +96,7 @@ class OSimCtrl:
     if screen == None:
       print "Did not find screen named %s" % self._screenName
     else:
-      print "Found screen %s" % screen.group(1)
+      print "Found screen %s" % screen
       
     print "OpenSimulator path: %s" % self._binaryPath
     
@@ -111,16 +112,21 @@ class OSimCtrl:
     screen = self.findScreen()
     
     if screen != None:
-      print >> sys.stderr, "Screen session %s already started." % (screen.group(1))
+      print >> sys.stderr, "Screen session %s already started." % (screen)
       return False
       
     chdir(self._binaryPath)
     
-    execCmd("%s -S %s -d -m %s %s.exe" % (self._screenPath, self._screenName, self._monoPath, self._componentName))
+    cmd = "%s %s.exe" % (self._monoPath, self._componentName)
+    
+    if opts.autorestart:
+      cmd = "bash -c 'set carryon=true; trap \"carryon=false\" SIGTERM; while $carryon; do %s; done'" % (cmd) 
+      
+    execCmd("%s -S %s -d -m %s" % (self._screenPath, self._screenName, cmd))
     
     screen = self.findScreen()
     if screen != None:
-      print "%s starting in screen instance %s" % (self._componentName, screen.group(1))
+      print "%s starting in screen instance %s" % (self._componentName, screen)
     else:
       print >> sys.stderr, "ERROR: %s did not start." % self._componentName
       return False
@@ -130,14 +136,21 @@ class OSimCtrl:
       
     return True
     
-  def stopComponent(self):
+  def stopComponent(self, opts):
     """Stop the given component.  Returns True on success, False if the component was already stopped or if there was a problem stopping."""
     screen = self.findScreen()    
     
     if screen == None:
       print >> sys.stderr, "No screen session named %s to stop" % self._screenName
       return False
-      
+
+    # If we're using the autorestart script then we need to sent it a SIGTERM first to stop it restarting
+    # the component
+    # FIXME: For now, always send a SIGTERM.  If we are not using the autorestart script then this will be received
+    # by the OpenSimulator mono instance instead, where it will be ignored.
+    autoRestartPid = int(execCmd("ps --ppid %s -o pid=" % screen.split(".")[0]))
+    os.kill(autoRestartPid, signal.SIGTERM)
+          
     execCmd("%s -S %s -p 0 -X stuff quit$(printf \r)" % (self._screenPath, self._screenName))
     
     timeElapsed = 0
@@ -164,7 +177,7 @@ class OSimCtrl:
     return self.startComponent(opts)
     
   def findScreen(self):
-    """Try to find the screen instance for this component.  Returns the screen name on success, None otherwise."""
+    """Try to find the screen instance for this component.  Returns the screen pid.name on success, None otherwise."""
     screenList = ""
     
     try:
@@ -174,7 +187,12 @@ class OSimCtrl:
       
     #print "screenList: %s" % screenList
     # TODO: Need to improve this for screens with ( in their name
-    return re.search("\s+(\d+\.%s)\s+\(" % self._screenName, screenList)
+    res = re.search("\s+(\d+\.%s)\s+\(" % self._screenName, screenList)
+    
+    if not res == None:
+      return res.group(1)
+    else:
+      return None
     
   def getScreenList(self):
     """Get a list of available screens directly from the screen command."""
@@ -192,6 +210,12 @@ def main(binaryPath, screenPath, monoPath, componentName, screenName):
     choices = commands.keys(), 
     help = "\n".join(["%s - %s" % (k, v['help']) for k, v in commands.iteritems()]))
   
+  parser.add_argument(
+    '-a',
+    '--autorestart',
+    help = "Automatically restart component if it crashes.  With this option, it can only be stopped via the stop command, not by manually attaching to the screen and shutting down the component.",
+    action = "store_true")
+      
   parser.add_argument(
     '-n', 
     '--noattach',