Python blockless subproccess input with constant output on Windows

Question

I am trying to run a command with subproccess and the _thread modules. The subproccess has a stream of output. To combat this I used two threads, one constantly prints new lines and the other is checking for input. When I pass the subproccess input through proc.stdin.write('Some string') it returns 1 and then I get no output. Communicate doesn't work as per most other questions I have read because it blocks waiting for the EOF although it does print the first line of the whatever was going to be returned. I saw a few solutions using 'pty' but it is not supported on Windows.

The file in the server folder is just a minecraft server if you want to try it yourself.

from subprocess import Popen,PIPE
import _thread
import sys
# asdf
proc = None
run = True
stdout = None
stdin = None


def getInput():
    global proc
    global run, stdin, stdout
    print("Proc inside the get input funct"+str(proc))
    inputs = input("Enter Something" + "\n")
    print("YOU ENTERED:", inputs)
    print("ATTEMPTING TO PIPE IT INTO THE CMD")
    run = True

    """----------------------------------------"""
    """        Works but blocks outputs        """
    """----------------------------------------"""
    # out,err=proc.communicate(bytes(inputs,'UTF-8'))
    # proc.stdin.flush()
    # print("Out is: "+out)




    """----------------------------------------"""
    """   Doesn't write but doesn't block      """
    """----------------------------------------"""
    # test = 0
    # test=proc.stdin.write(bytes(inputs,'UTF-8'))
    # print(test)
    # proc.stdin.flush()


def execute(command):
    global proc, stdin, stdout
    proc = Popen(command, cwd='C://Users//Derek//Desktop//server//',stdin=PIPE,stdout=PIPE,stderr=stdout, shell=True)
    lines_iterator = iter(proc.stdout.readline, "")
    print("Proc inside of the execute funct:"+str(proc))
    # print(lines_iterator)
    for line in lines_iterator:
        # print(str(line[2:-1]))
        # if line.decode('UTF-8') != '':
        print(line[:-2].decode('UTF-8')),  # yield line
        sys.stdout.flush()


threadTwo = _thread.start_new_thread(execute, (["java", "-jar", "minecraft_server.jar"], ))

while 1:
    if run and proc!=None:
        run = False
        threadOne = _thread.start_new_thread(getInput, ( ))

    pass

Answer 1

proc.communicate() waits for the subprocess to finish therefore it can be used at most once – you can pass all input at once and get all the output after the child process exits.

If you are not modifying input/output then you do not need to redirect subprocess' stdin/stdout.

To feed input to a subprocess in a background thread and to print its output as soon as it arrives line-by-line:

#!/usr/bin/env python3
import errno
from io import TextIOWrapper
from subprocess import Popen, PIPE
from threading import Thread

def feed(pipe):
    while True:
        try: # get input
            line = input('Enter input for minecraft')
        except EOFError:
            break # no more input
        else:
            # ... do something with `line` here

            # feed input to pipe
            try:
                print(line, file=pipe)
            except BrokenPipeError:
                break # can't write to pipe anymore
            except OSError as e:
                if e.errno == errno.EINVAL:
                    break  # same as EPIPE on Windows
                else:
                    raise # allow the error to propagate

    try:
        pipe.close() # inform subprocess -- no more input
    except OSError:
        pass # ignore

with Popen(["java", "-jar", "minecraft_server.jar"],
           cwd=r'C:\Users\Derek\Desktop\server',
           stdin=PIPE, stdout=PIPE, bufsize=1) as p, \
     TextIOWrapper(p.stdin, encoding='utf-8', 
                   write_through=True, line_buffering=True) as text_input:
    Thread(target=feed, args=[text_input], daemon=True).start()
    for line in TextIOWrapper(p.stdout, encoding='utf-8'):
        # ... do something with `line` here
        print(line, end='')

Note about p.stdin:

1. print() adds a newline at the end of each line. It is necessary because input() strips the newline
2. p.stdin.flush() is called after each line (line_buffering=True)

The output from minecraft may be delayed until its stdout buffer is flushed.

If you have nothing to add around the "do something with line here" comments then do not redirect corresponding pipes (ignoring character encoding issues for a moment).

TextIOWrapper uses the universal newline mode by default. Specify newline parameter explicitly if you do not want that.