Why? Because the processes were filling up their std buffers and were blocked writing to stdout.
Working code. With enough comments to get at the cause of the problem.
#!/usr/bin/python
import pexpect
import threading
import time
# Consumes output of some subprocess in a background thread
class OutletThread(threading.Thread):
# obj: pexpect.spawn object
# delay: time (in seconds) to wait between two reads
def __init__(self, obj, delay = 1):
threading.Thread.__init__(self)
self.obj = obj
self.on = True
self.delay = delay
# main
def run(self):
while self.on:
try:
self.obj.read_nonblocking(size=2048, timeout=0)
except pexpect.TIMEOUT:
pass
time.sleep(self.delay)
# Brief
# Print expected output
# Arguments
# obj: pexpect.spawn object
# s: expected string
# Exceptions
# pexpect.TIMEOUT if default timeout is not met
def pp(obj, s):
obj.expect(s)
print obj.before + obj.after
print("Setting up the environment...")
print("Registry...")
r = pexpect.spawn('./registry.sh')
# Hold a moment for Java RMI Registry to start.
# So that controller can connect properly.
time.sleep(1)
print("Controller...")
c = pexpect.spawn ('./controller.sh')
pp(c, '>')
c_rlt = OutletThread(c)
c_rlt.start()
print("Node...")
n = pexpect.spawn ('./node.sh')
pp(n, '>')
n_rlt = OutletThread(n)
n_rlt.start()
print("Client...")
cl = pexpect.spawn ('./client.sh')
pp(cl, '>')
cl_rlt = OutletThread(cl)
cl_rlt.start()
raw_input("Press Enter to run 'info' at the controller...")
# Disable controller outlet
c_rlt.on = False
# Run command
c.sendline('info')
# Expect output
pp(c, '>')
# Enable controller outlet
c_rlt.on = True
raw_input("Press Enter to exit...")
# Disable and join all outlet threads
c_rlt.on = False
c_rlt.join()
n_rlt.on = False
n_rlt.join()
cl_rlt.on = False
cl_rlt.join()
# Terminate all spawned processes
cl.sendintr()
n.sendintr()
c.sendintr()
r.sendintr()
Output.
Setting up the environment...
Registry...
Controller...
INFO ds.controller.Main - Starting Controller
INFO ds.controller.Main - Registered remote object
controller>
Node...
INFO ds.node.Main - Starting Node
INFO ds.node.Main - Base folder folders/node
INFO ds.node.Main - ServerSocket created
INFO ds.node.Main - Registered remote object as NodeServer-1385415881716
INFO ds.node.Main - Created folder folders/node/NodeServer-1385415881716
DEBUG ds.node.Commands - Connected to controller
INFO ds.node.rmi.NodeFileServerImpl - Start listening on ServerSocketPort
INFO ds.node.rmi.NodeFileServerImpl - Returning ServerSocketPort
DEBUG ds.shared.socket.ServerSocketHandler - Waiting for incomming connections
DEBUG ds.shared.socket.ServerSocketHandler - Accepted connection
DEBUG ds.shared.socket.ClientSocketHandler - Connection ID 0
DEBUG ds.shared.socket.ServerSocketHandler - Set clientHandler: ds.shared.socket.ClientSocketHandler@a9c8620
node>
Client...
INFO ds.client.Main - Starting Client
INFO ds.client.Main - Base folder folders/client
INFO ds.client.Main - Registered remote object as Client-1385415882026
DEBUG ds.client.Commands - Connected to controller.
DEBUG ds.client.Commands - Connecting to 192.168.1.8:50690
DEBUG ds.client.Commands - Opened socket connection.
client>
Press Enter to run 'info' at the controller...
info
DS Controller, version 0.1 (Beta)
Connected nodes:
* NodeServer-1385415881716
controller>
Press Enter to exit...
Fedors-MacBook-Pro:distributed ted$ ../expect/nv.py
Setting up the environment...
Registry...
Controller...
INFO ds.controller.Main - Starting Controller
INFO ds.controller.Main - Registered remote object
controller>
Node...
INFO ds.node.Main - Starting Node
INFO ds.node.Main - Base folder folders/node
INFO ds.node.Main - ServerSocket created
INFO ds.node.Main - Registered remote object as NodeServer-1385417858283
INFO ds.node.Main - Created folder folders/node/NodeServer-1385417858283
DEBUG ds.node.Commands - Connected to controller
INFO ds.node.rmi.NodeFileServerImpl - Start listening on ServerSocketPort
INFO ds.node.rmi.NodeFileServerImpl - Returning ServerSocketPort
DEBUG ds.shared.socket.ServerSocketHandler - Waiting for incomming connections
DEBUG ds.shared.socket.ServerSocketHandler - Accepted connection
DEBUG ds.shared.socket.ClientSocketHandler - Connection ID 0
DEBUG ds.shared.socket.ServerSocketHandler - Set clientHandler: ds.shared.socket.ClientSocketHandler@1d672476
node>
Client...
INFO ds.client.Main - Starting Client
INFO ds.client.Main - Base folder folders/client
INFO ds.client.Main - Registered remote object as Client-1385417858595
DEBUG ds.client.Commands - Connected to controller.
DEBUG ds.client.Commands - Connecting to 192.168.1.8:50883
DEBUG ds.client.Commands - Opened socket connection.
client>
Press Enter to run 'info' at the controller...
info
DS Controller, version 0.1 (Beta)
Connected nodes:
* NodeServer-1385417858283
controller>
Press Enter to exit...
Assets.
- Python 2.7.5
- Pexpect 3.0
- threading and time libraries
Links.
- Pexpect 3.0 documentation
- Python Multithreaded Programming
- 16.2. threading — Higher-level threading interface — Python v2.7.6 documentation
Big thanks to Thomas Kluyver, other pexpect enthusiasts and stackoverflow team.