From 65964bfca85de89855cd679120dbb4bf42d3c92f Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 20 May 2014 00:41:34 -0700 Subject: [PATCH 01/11] Sketching out shell usage doc. For remote pipes. --- doc/shell.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 doc/shell.md diff --git a/doc/shell.md b/doc/shell.md new file mode 100644 index 0000000..da6d787 --- /dev/null +++ b/doc/shell.md @@ -0,0 +1,45 @@ +Using webpipe with the shell +============================ + +The original idea behind webpipe was to use it with an R client. But using it +at the shell opens up a lot of possibilities, e.g. for system administration +and performance analysis. + +# run the server that listens on port 8988 + +$ wp run + +# hm I don't want to run another server... + +# Run EITHER + +$ wp run-recv +# listen on port 8988 for local filenames +# also listen on 8987 for remote + +# Run a socat server on 8987, which pipes to recv, then writes the file, then +# sends filename to 9899. + + +# Now you can send stuff locally to 8987. "wp-stub show" ? Or send? + + # copy +$ wp scp-stub example.com + +$ wp ssh example.com # Local 8987 tunnel + +example.com$ wp-stub show foo.png + +Now foo.png appears in the browser on localhost:8989. + + +Even further generalization: + +You want not just the xrender endpoint, but the raw server endpoint? Hm. +Allow interposition at any point... + +The use case is where you run the render pipelines? Running render pipelines +remotely, but server locally. If you want to be secure, and don't want to +expose an HTTP server. + + From 745c7361ba03e228d577a3fe75d94d58bbcfb47b Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 20 May 2014 00:48:46 -0700 Subject: [PATCH 02/11] Solution for nc/socat issue. --- wp.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/wp.sh b/wp.sh index 877f966..3c42ed8 100755 --- a/wp.sh +++ b/wp.sh @@ -155,6 +155,14 @@ run() { export PYTHONUNBUFFERED=1 + # TODO: Add xrender --listen-port 8988. If port is specified, then instead of + # reading from stdin, we listen on a port. + # + # nc servers don't reliably work the same way on all machines. socat isn't + # installed. + # + # We might want recv --listen-port too, but maybe later. And even server + # --listen-port. socat-listen 8988 \ | xrender $INPUT_DIR $session \ | serve serve $session From 4d09e2caa57a3f7161e8c7e6de34f2701cc76147 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 20 May 2014 00:53:36 -0700 Subject: [PATCH 03/11] Minor cleanup of xrender. --- webpipe/xrender.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/webpipe/xrender.py b/webpipe/xrender.py index 89ed3f8..049f4b6 100755 --- a/webpipe/xrender.py +++ b/webpipe/xrender.py @@ -5,30 +5,18 @@ A filter that reads filenames from stdin, and prints HTML directories on stdout. -File types: +TODO: Make this usable as a library too? So in the common case you can have a +single process. -- .png -> inline HTML images (data URIs) -- .csv -> table +File types: Next: -- .script -- type script for Shell session - - a configurable prefix, like "hostname; whoami;" etc. would be useful. - .grep -- grep results - But then do we have to copy a ton of files over? - xrender needs have access to the original directory. - and the command -Ideas: - -- .url file -> previews of URLs? -- .foo-url -> previews of a certain page? - -Should this call a shell script then? Or it could be a shell script? The -function will use the tnet tool? - -TODO: Make this usable as a library too? So in the common case you can have a -single process. Plugins ------- @@ -56,8 +44,6 @@ - maybe you have to dereference the link. """ -import cgi -import errno import json import os import re @@ -189,6 +175,7 @@ def main(argv): sys.stdout.write(tnet.dump_line(header)) while True: + # TODO: Read from Queue? Could be pipe or socket. line = sys.stdin.readline() if not line: break From 95a75197f9fc78ca84bbba12425cb63d01ce05b2 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 20 May 2014 01:08:47 -0700 Subject: [PATCH 04/11] More implementation notes. --- doc/shell.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/doc/shell.md b/doc/shell.md index da6d787..72d8647 100644 --- a/doc/shell.md +++ b/doc/shell.md @@ -43,3 +43,40 @@ remotely, but server locally. If you want to be secure, and don't want to expose an HTTP server. +Pipeline implementation options: + +- threads and Queue() +- coroutines + - it's synchronous dataflow, so you don't need queues really. + +Add --listen for every stage? So you can listen on recv input, xrender input, +or server input port (and listen on HTTP serving port too) + +What's the command line syntax? You can run: + +- serve +- xrender and serve +- recv xrender and serve + +And you can run other parts on the remote machine. + +- send +- xrender then send + +So pipelines look like this, where == is a TCP socket separating machines. | +is a pipe, or possibly in-process Queue/channel. + +xrender | serve (local machine only) + +send == recv | xrender | serve + +xrender | send == recv | serve + +NOTE: send does not currently handle directories, or the combo of file + +directory that webpipe uses. I guess we write the file last for this reason. + + +What does the client look like? Difference between show and send? I think you +want to use show everywhere. Both local and remote. Ports are the same. +"show" goes to port 8988? "send" goes to port 8987. + From 339e00811f2b9f03f84e4369be19241ce0c5cb94 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 20 May 2014 01:43:55 -0700 Subject: [PATCH 05/11] Make note about coroutine architecture. --- doc/shell.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/shell.md b/doc/shell.md index 72d8647..08e527a 100644 --- a/doc/shell.md +++ b/doc/shell.md @@ -48,6 +48,10 @@ Pipeline implementation options: - threads and Queue() - coroutines - it's synchronous dataflow, so you don't need queues really. + - BUT: if each stage has input from a real socket, and the previous stage, + can you do that with Python coroutines? can you do fan in? + I think so. Call stage.send() from two places. + Multiple network ports need a select() loop. Add --listen for every stage? So you can listen on recv input, xrender input, or server input port (and listen on HTTP serving port too) From 314f2fbc4bf52a60d8ac0976bc74a09e2157b887 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 20 May 2014 02:28:59 -0700 Subject: [PATCH 06/11] More ideas. --- doc/shell.md | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/doc/shell.md b/doc/shell.md index 08e527a..d5c3530 100644 --- a/doc/shell.md +++ b/doc/shell.md @@ -1,6 +1,8 @@ Using webpipe with the shell ============================ +Or "Bringing back graphical computing for headless servers" + The original idea behind webpipe was to use it with an R client. But using it at the shell opens up a lot of possibilities, e.g. for system administration and performance analysis. @@ -28,11 +30,13 @@ $ wp scp-stub example.com $ wp ssh example.com # Local 8987 tunnel +# TODO: Also need to test two tunnels. Can we use the same port? What about +# with big files? + example.com$ wp-stub show foo.png Now foo.png appears in the browser on localhost:8989. - Even further generalization: You want not just the xrender endpoint, but the raw server endpoint? Hm. @@ -84,3 +88,34 @@ What does the client look like? Difference between show and send? I think you want to use show everywhere. Both local and remote. Ports are the same. "show" goes to port 8988? "send" goes to port 8987. +Ports: +- 8987 for recv input +- 8988 for xrender input +- TODO: change is to it's 8980 8981 8982? Three consecutive ports + WEBPIPE_RECV_PORT + WEBPIPE_XRENDER_PORT + WEBPIPE_HTML_PORT + Need this because those ports might be used on the machine. + +- 8989 is webpipe HTTP server +- 8900 is for latch HTTP + + +More advanced idea +------------------ + +Along the lines of "bringing back graphical computing". + +- What if you want a "live top" or something. Or a "live pstree". A visualization. + +Then you aren't just sending files back. You want a client program that prints +a textual protocol to stdout for updates. Like it will print CPU usage lines +or something. + +And then you want to pipe that directly to JS and visualize it? + +The easiest demo is some kind of time series. You scp a little shell/Python +script to a server. And then it will output a time series. + + + From 6b9fe067a0050c7966743ae112083cddd3660479 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 22 May 2014 15:26:56 -0700 Subject: [PATCH 07/11] Factor out function. --- webpipe/xrender.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/webpipe/xrender.py b/webpipe/xrender.py index 049f4b6..466c537 100755 --- a/webpipe/xrender.py +++ b/webpipe/xrender.py @@ -135,13 +135,7 @@ def GetPluginBin(self, file_type): return None -def main(argv): - """Returns an exit code.""" - - # NOTE: This is the input base path. We just join them with the filenames on - # stdin. - in_dir = argv[1] - out_dir = argv[2] +def Loop(in_dir, out_dir): # TODO: # - input is a single line for now. Later it could be a message, if you want # people to specify an explicit file type. I guess that can be done with a @@ -292,6 +286,16 @@ def main(argv): return 0 +def main(argv): + """Returns an exit code.""" + # NOTE: This is the input base path. We just join them with the filenames on + # stdin. + in_dir = argv[1] + out_dir = argv[2] + + Loop(in_dir, out_dir) + + if __name__ == '__main__': try: sys.exit(main(sys.argv)) From 6307bd2b7e4265e9390422b9119840966e52e714 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 22 May 2014 15:40:53 -0700 Subject: [PATCH 08/11] "Pipe" stdin to coroutine. --- webpipe/xrender.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/webpipe/xrender.py b/webpipe/xrender.py index 466c537..94721f9 100755 --- a/webpipe/xrender.py +++ b/webpipe/xrender.py @@ -135,7 +135,11 @@ def GetPluginBin(self, file_type): return None -def Loop(in_dir, out_dir): +def PluginDispatchLoop(in_dir, out_dir): + """ + Coroutine that passes its input to a rendering plugin. + """ + # TODO: # - input is a single line for now. Later it could be a message, if you want # people to specify an explicit file type. I guess that can be done with a @@ -169,10 +173,8 @@ def Loop(in_dir, out_dir): sys.stdout.write(tnet.dump_line(header)) while True: - # TODO: Read from Queue? Could be pipe or socket. - line = sys.stdin.readline() - if not line: - break + # NOTE: Coroutine + line = yield # TODO: If file contains punctuation, escape it to be BOTH shell and HTML # safe, and then MOVE It to ~/webpipe/safe-name @@ -283,8 +285,6 @@ def Loop(in_dir, out_dir): counter += 1 - return 0 - def main(argv): """Returns an exit code.""" @@ -293,7 +293,23 @@ def main(argv): in_dir = argv[1] out_dir = argv[2] - Loop(in_dir, out_dir) + # PluginDispatchLoop is a coroutine. It takes items to render on stdin. + loop = PluginDispatchLoop(in_dir, out_dir) + + # "prime" the coroutine. + next(loop) + + while True: + line = sys.stdin.readline() + if not line: # EOF + break + + try: + loop.send(line) + except StopIteration: + break + + return 0 if __name__ == '__main__': From 940c217fbc6e7a153fc4b7dbd36de907f42ebb63 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 22 May 2014 16:21:42 -0700 Subject: [PATCH 09/11] Implement TCP server to replace unportable netcat. --- webpipe/xrender.py | 89 ++++++++++++++++++++++++++++++++++------- webpipe/xrender_test.py | 15 +++++++ 2 files changed, 90 insertions(+), 14 deletions(-) diff --git a/webpipe/xrender.py b/webpipe/xrender.py index 94721f9..370e8b8 100755 --- a/webpipe/xrender.py +++ b/webpipe/xrender.py @@ -47,6 +47,7 @@ import json import os import re +import socket import subprocess import sys import time @@ -173,12 +174,10 @@ def PluginDispatchLoop(in_dir, out_dir): sys.stdout.write(tnet.dump_line(header)) while True: - # NOTE: Coroutine - line = yield - + # NOTE: This is a coroutine. + filename = yield # TODO: If file contains punctuation, escape it to be BOTH shell and HTML # safe, and then MOVE It to ~/webpipe/safe-name - filename = line.strip() # NOTE: Right now, this allows absolute paths too. input_path = os.path.join(in_dir, filename) @@ -286,6 +285,63 @@ def PluginDispatchLoop(in_dir, out_dir): counter += 1 +def Lines(f, target): + """ + Read lines from the given file object and deliver to the given coroutine. + """ + target.next() # "prime" coroutine + + while True: + line = f.readline() + if not line: # EOF + break + + filename = line.strip() + try: + target.send(filename) + except StopIteration: + break + + +# Max 1 megabyte. We close it anyway. +BUF_SIZE = 1 << 20 + +# TODO: Should this be moved to "util" so other stages can use it? +def TcpServer(port, target): + """ + Listen on a port, and read (small) values send from different clients, and + deliver them to the given coroutine. + + This basically replaces "nc -k -l 8000 Date: Thu, 22 May 2014 17:13:50 -0700 Subject: [PATCH 10/11] Hookup the TCP server to xrender. --- webpipe-test.sh | 8 ++++++++ webpipe/xrender.py | 45 +++++++++++++++++++++++------------------ webpipe/xrender_test.py | 2 ++ wp-test.sh | 2 ++ 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/webpipe-test.sh b/webpipe-test.sh index 31f799b..23e6112 100755 --- a/webpipe-test.sh +++ b/webpipe-test.sh @@ -5,6 +5,9 @@ set -o nounset +# TODO: make this a proper dep +. ~/hg/taste/taste.sh + # # Components # @@ -83,6 +86,11 @@ write-demo() { # Tests # +test-xrender-badport() { + xrender -p invalid + check $? -eq 1 +} + test-xrender() { xrender ~/webpipe/input ~/webpipe/s/webpipe-test < +set -o nounset + # TODO: make this a proper dep . ~/hg/taste/taste.sh From 3a48db872c62bf5c59e0d8e3b9b62e857bd4c338 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 22 May 2014 17:20:47 -0700 Subject: [PATCH 11/11] Hook up the new TCP server as the default for 'wp run'. --- wp.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/wp.sh b/wp.sh index 3c42ed8..3ac9eb2 100755 --- a/wp.sh +++ b/wp.sh @@ -121,6 +121,8 @@ serve() { $THIS_DIR/webpipe/serve.py "$@" } +# Better/more portable server than nc. Should only be used when we don't care +# about running on Mac. socat-listen() { local port=$1 # -u: unidirection (