I am trying to use twisted spawnProcess to replicate the behavior of something like this:
cat <input.txt | wc -w
This is just an example of two commands, in reality I have my own processes (say python or bash scripts or external programs) where each process reads from stdin and writes to stdout. Just like the above example, I want to pipe stdout from one process to stdin of another, and I want to do this using spawnProcess. I used some hints here:
Twisted pipe two processes with spawnProcess
but I can't get it to work. It just hangs when reading from stdin on the second spawnProcess protocol. My code is below.What am I doing wrong? How exactly can I achieve this objective? Is it better to call the second spawnProcess from within the first?
#!/usr/bin/env python
from twisted.internet import protocol
from twisted.internet import reactor
import re
import os
import sys
class CatPP(protocol.ProcessProtocol):
def __init__(self,input_data):
self.input_data=input_data
self.data = ""
def connectionMade(self):
print "connectionMade in CatPP! Now writing to stdin of cat"
print " writing this data: %s" % self.input_data
self.transport.write(self.input_data+'\n')
print " closing stdin"
self.transport.closeStdin() # tell them we're done
print " stdin closed"
def outReceived(self, data):
print "outReceived from cat! with %d bytes!" % len(data)
self.data = self.data + data
print " received this: %s" % self.data
def errReceived(self, data):
print "errReceived from cat! with %d bytes!" % len(data)
def inConnectionLost(self):
print "inConnectionLost for cat! stdin is closed! (we probably did it)"
def outConnectionLost(self):
print "outConnectionLost for cat! The child closed their stdout!"
# now is the time to examine what they wrote
print "I saw cat write this:", self.data
def errConnectionLost(self):
print "errConnectionLost for cat! The child closed their stderr."
def processExited(self, reason):
print "processExited for cat, status %d" % (reason.value.exitCode,)
def processEnded(self, reason):
print "processEnded for cat, status %d" % (reason.value.exitCode,)
class WcPP(protocol.ProcessProtocol):
def __init__(self):
self.data = ""
def connectionMade(self):
print "connectionMade! Now reading from pipe to get stdin for wp"
print " reading from stdin"
txt = sys.stdin.read()
print " Read this from stdin: %s" % (txt,)
self.transport.write(txt)
self.transport.closeStdin() # tell them we're done
def outReceived(self, data):
print "outReceived from cat! with %d bytes!" % len(data)
self.data = self.data + data
def errReceived(self, data):
print "errReceived from cat! with %d bytes!" % len(data)
def inConnectionLost(self):
print "inConnectionLost for cat! stdin is closed! (we probably did it)"
def outConnectionLost(self):
print "outConnectionLost for cat! The child closed their stdout!"
# now is the time to examine what they wrote
print "Final output:", self.data
#(dummy, lines, words, chars, file) = re.split(r'\s+', self.data)
#print "I saw %s lines" % lines
def errConnectionLost(self):
print "errConnectionLost for cat! The child closed their stderr."
def processExited(self, reason):
print "processExited for cat, status %d" % (reason.value.exitCode,)
def processEnded(self, reason):
print "processEnded for cat, status %d" % (reason.value.exitCode,)
reactor.stop()
readPipe, writePipe = os.pipe()
handle=open('junkin.txt','r')
cat_txt=handle.read()
handle.close()
pp1 = CatPP(cat_txt)
pp2 = WcPP()
reactor.spawnProcess(pp1, "cat", ["cat"], {}, childFDs={1: writePipe})
reactor.spawnProcess(pp2, "wc", ["wc", "-w"], {},childFDs={0: readPipe})
reactor.run()
try:
os.close(readPipe)
except:
print "Exception closing readPipe"
try:
os.close(writePipe)
except:
print "Exception closing writePipe"
Here is a working example.
In the case of piping through cat | wc, spawnProcess duplicates the pipes so you need to close them.
from twisted.internet import protocol
from twisted.internet import reactor
import os
class Writer(protocol.ProcessProtocol):
def __init__(self, data):
self.data = data
def connectionMade(self):
print "Writer -- connection made"
self.transport.writeToChild(0, self.data)
self.transport.closeChildFD(0)
def childDataReceived(self, fd, data):
pass
def processEnded(self, status):
pass
class Reader(protocol.ProcessProtocol):
def __init__(self):
pass
def connectionMade(self):
print "Reader -- connection made"
pass
def childDataReceived(self, fd, data):
print "Reader -- childDataReceived"
self.received = data
def processEnded(self, status):
print "process ended, got:", self.received
class WriteRead(protocol.ProcessProtocol):
def __init__(self, data):
self.data = data
def connectionMade(self):
self.transport.writeToChild(0, self.data)
self.transport.closeChildFD(0)
def childDataReceived(self, fd, data):
self.received = data
print "got data:", data
def processEnded(self, status):
print "process ended - now what?"
def test1(data):
# just call wc
p2 = reactor.spawnProcess(WriteRead(data), "wc", ["wc"], env=None, childFDs={0: "w", 1: "r"})
reactor.run()
def test2(data):
rfd, wfd = os.pipe()
p1 = reactor.spawnProcess(Writer(data), "cat", ["cat"], env=None, childFDs={0:"w", 1: wfd })
p2 = reactor.spawnProcess(Reader(), "wc", ["wc", "-w"], env=None, childFDs={0: rfd, 1: "r"})
os.close(rfd)
os.close(wfd)
reactor.run()
test2("this is a test")
Marc
Ok I figured out how to do it but without using the pipes approach from Twisted pipe two processes with spawnProcess, which I never got to work. Instead I used a class ChainableProcessProtocol that takes another ChainableProcessProtocol as an argument.This way you can chain these together, with the output of a prior protocol writing to the stdin of the next protocol. This chaining stops when the next_protocol is None. The final output is the data from the final protocol. Here is my example:
#!/usr/bin/env python
from twisted.internet import protocol
from twisted.internet import reactor, defer
import re
import os
import sys
import json
def shutdown(x):
print "Shutdown called"
reactor.stop()
class ChainableProcessProtocol(protocol.ProcessProtocol):
def __init__(self,cmd,cmdargs,input_data,next_protocol):
self.cmd=cmd
self.cmdargs=cmdargs
self.input_data=input_data
self.next_protocol=next_protocol
self.data = ""
def set_input_data(self,new_input_data):
self.input_data=new_input_data
def connectionMade(self):
print "connectionMade in %s! Now writing to stdin of cat" % self.cmd
print " writing this data: %s" % self.input_data
self.transport.write(self.input_data+'\n')
print " closing stdin"
self.transport.closeStdin() # tell them we're done
print " stdin closed"
def outReceived(self, data):
print "outReceived from %s! with %d bytes!" % (self.cmd,len(data))
self.data = self.data + data
print " received this: %s" % self.data
def errReceived(self, data):
print "errReceived from %s! with %d bytes!" % (self.cmd,len(data))
def inConnectionLost(self):
print "inConnectionLost for %s! stdin is closed! (we probably did it)" %(self.cmd,)
def outConnectionLost(self):
print "outConnectionLost for %s! The child closed their stdout!" %(self.cmd,)
# now is the time to examine what they wrote
print "I saw %s write this: %s" % (self.cmd,self.data)
#
# cmd is done, now write to next_protocol if set
#
if self.next_protocol:
print "Calling chained protocol"
self.next_protocol.set_input_data(self.data)
npcmd=self.next_protocol.cmd
npcmdargs=self.next_protocol.cmdargs
print "npcmd is %s" % (npcmd,)
print "npcmdargs is %s" % (json.dumps(npcmdargs),)
reactor.spawnProcess(self.next_protocol, npcmd, npcmdargs, {})
else:
print "No chained protocol"
def errConnectionLost(self):
print "errConnectionLost for %s! The child closed their stderr." % (self.cmd,)
def processExited(self, reason):
print "processExited for %s, status %d" % (self.cmd,reason.value.exitCode,)
def processEnded(self, reason):
print "processEnded for %s, status %d" % (self.cmd,reason.value.exitCode,)
handle=open('junkin.txt','r')
in_txt=handle.read()
handle.close()
#
# Create the last protocol first because earlier protocol(s) need it
#
pp2 = ChainableProcessProtocol("wc",["wc","-w"],'',None)
#
# The first process takes an instance of the second process as an argument
#
pp1 = ChainableProcessProtocol("cat",["cat"],in_txt,pp2)
#
# before using spawnProcess, lets create a deferred to shut down the reactor in 2 seconds
# This should give us enough time. This is only so our test script does not run forever
#
d=defer.Deferred()
d.addCallback(shutdown)
reactor.callLater(2, d.callback, '')
#
# Now spawn the first process
#
reactor.spawnProcess(pp1, pp1.cmd, pp1.cmdargs, {})
reactor.run()
print "Final output is data in pp2: %s" % (pp2.data.strip(),)
print "Done!"
来源：https://stackoverflow.com/questions/27513516/twisted-spawnprocess-send-output-of-one-process-to-input-of-another