Merge branch 'socat-remove'

doc/shell.md

@@ -0,0 +1,121 @@
+Using webpipe with the shell
+============================
+
+Or "Bringing back graphical computing for headless servers"
+
+The original idea behind webpipe was to use it with an R client.  But using it
+at the shell opens up a lot of possibilities, e.g. for system administration
+and performance analysis.
+
+# run the server that listens on port 8988
+
+$ wp run
+
+# hm I don't want to run another server...
+
+# Run EITHER
+
+$ wp run-recv
+# listen on port 8988 for local filenames
+# also listen on 8987 for remote
+
+# Run a socat server on 8987, which pipes to recv, then writes the file, then
+# sends filename to 9899.
+
+
+# Now you can send stuff locally to 8987.  "wp-stub show" ?  Or send?
+
+ # copy
+$ wp scp-stub example.com
+
+$ wp ssh example.com    # Local 8987 tunnel
+
+# TODO: Also need to test two tunnels.  Can we use the same port?  What about
+# with big files?
+
+example.com$ wp-stub show foo.png
+
+Now foo.png appears in the browser on localhost:8989.
+
+Even further generalization:
+
+You want not just the xrender endpoint, but the raw server endpoint?  Hm.
+Allow interposition at any point...
+
+The use case is where you run the render pipelines?  Running render pipelines
+remotely, but server locally.  If you want to be secure, and don't want to
+expose an HTTP server.
+
+
+Pipeline implementation options:
+
+- threads and Queue()
+- coroutines
+  - it's synchronous dataflow, so you don't need queues really.
+  - BUT: if each stage has input from a real socket, and the previous stage,
+    can you do that with Python coroutines?  can you do fan in?
+    I think so.  Call stage.send() from two places.
+    Multiple network ports need a select() loop.
+
+Add --listen for every stage?  So you can listen on recv input, xrender input,
+or server input port (and listen on HTTP serving port too)
+
+What's the command line syntax?  You can run:
+
+- serve
+- xrender and serve
+- recv xrender and serve
+
+And you can run other parts on the remote machine.
+
+- send
+- xrender then send
+
+So pipelines look like this, where == is a TCP socket separating machines.  |
+is a pipe, or possibly in-process Queue/channel.
+
+xrender | serve (local machine only)
+
+send == recv | xrender | serve
+
+xrender | send == recv | serve
+
+NOTE: send does not currently handle directories, or the combo of file +
+directory that webpipe uses.  I guess we write the file last for this reason.
+
+
+What does the client look like?  Difference between show and send?  I think you
+want to use show everywhere.  Both local and remote.  Ports are the same.
+"show" goes to port 8988?  "send" goes to port 8987.
+
+Ports:
+- 8987 for recv input
+- 8988 for xrender input
+- TODO: change is to it's 8980 8981 8982?  Three consecutive ports
+  WEBPIPE_RECV_PORT
+  WEBPIPE_XRENDER_PORT
+  WEBPIPE_HTML_PORT
+  Need this because those ports might be used on the machine.
+
+- 8989 is webpipe HTTP server
+- 8900 is for latch HTTP
+
+
+More advanced idea
+------------------
+
+Along the lines of "bringing back graphical computing".
+
+- What if you want a "live top" or something.  Or a "live pstree".  A visualization.
+
+Then you aren't just sending files back.  You want a client program that prints
+a textual protocol to stdout for updates.  Like it will print CPU usage lines
+or something.
+
+And then you want to pipe that directly to JS and visualize it?
+
+The easiest demo is some kind of time series.  You scp a little shell/Python
+script to a server.  And then it will output a time series.
+
+
+

webpipe-test.sh

@@ -5,6 +5,9 @@
 
 set -o nounset
 
+# TODO: make this a proper dep
+. ~/hg/taste/taste.sh
+
 #
 # Components
 #
@@ -83,6 +86,11 @@ write-demo() {
 # Tests
 #
 
+test-xrender-badport() {
+  xrender -p invalid
+  check $? -eq 1
+}
+
 test-xrender() {
   xrender ~/webpipe/input ~/webpipe/s/webpipe-test <<EOF
 $PWD/plugins/txt/testdata/example.txt

webpipe/xrender.py

@@ -5,30 +5,18 @@
 A filter that reads filenames from stdin, and prints HTML directories on
 stdout.
 
-File types:
+TODO: Make this usable as a library too?  So in the common case you can have a
+single process.
 
-- .png -> inline HTML images (data URIs) 
-- .csv -> table
+File types:
 
 Next:
 
-- .script -- type script for Shell session
-  - a configurable prefix, like "hostname; whoami;" etc. would be useful.
 - .grep -- grep results
   - But then do we have to copy a ton of files over?
   - xrender needs have access to the original directory.
   - and the command
 
-Ideas:
-
-- .url file -> previews of URLs?
-- .foo-url -> previews of a certain page?
-
-Should this call a shell script then?  Or it could be a shell script?  The
-function will use the tnet tool?
-
-TODO: Make this usable as a library too?  So in the common case you can have a
-single process.
 
 Plugins
 -------
@@ -49,18 +37,19 @@
 - provide original file for download (in most cases)
 
 - zero copy
-  - if you make a symlink, then the plugin can read that stuff, create a summary
+  - if you make a symlink, then the plugin can read that stuff, create a
+    summary
   - and then can it output a *capability* for the server to serve files
     anywhere on the file system?
     - or perhaps the symlink is enough?  well it could change.
     - maybe you have to dereference the link.
 """
 
-import cgi
-import errno
+import getopt
 import json
 import os
 import re
+import socket
 import subprocess
 import sys
 import time
@@ -149,13 +138,11 @@ def GetPluginBin(self, file_type):
     return None
 
 
-def main(argv):
-  """Returns an exit code."""
+def PluginDispatchLoop(in_dir, out_dir):
+  """
+  Coroutine that passes its input to a rendering plugin.
+  """
 
-  # NOTE: This is the input base path.  We just join them with the filenames on
-  # stdin.
-  in_dir = argv[1]
-  out_dir = argv[2]
   # TODO:
   # - input is a single line for now.  Later it could be a message, if you want
   # people to specify an explicit file type.  I guess that can be done with a
@@ -189,13 +176,10 @@ def main(argv):
   sys.stdout.write(tnet.dump_line(header))
 
   while True:
-    line = sys.stdin.readline()
-    if not line:
-      break
-
+    # NOTE: This is a coroutine.
+    filename = yield
     # TODO: If file contains punctuation, escape it to be BOTH shell and HTML
     # safe, and then MOVE It to ~/webpipe/safe-name
-    filename = line.strip()
 
     # NOTE: Right now, this allows absolute paths too.
     input_path = os.path.join(in_dir, filename)
@@ -302,12 +286,101 @@ def main(argv):
 
     counter += 1
 
+
+def Lines(f, target):
+  """
+  Read lines from the given file object and deliver to the given coroutine.
+  """
+  target.next()  # "prime" coroutine
+
+  while True:
+    line = f.readline()
+    if not line:  # EOF
+      break
+
+    filename = line.strip()
+    try:
+      target.send(filename)
+    except StopIteration:
+      break
+
+
+# Max 1 megabyte.  We close it anyway.
+BUF_SIZE = 1 << 20
+
+# TODO: Should this be moved to "util" so other stages can use it?
+def TcpServer(port, target):
+  """
+  Listen on a port, and read (small) values send from different clients, and
+  deliver them to the given coroutine.
+
+  This basically replaces "nc -k -l 8000 </dev/null", which is not portable
+  across machines (especially OS X).
+  """
+  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+  # When we die, let other processes use port immediately.
+  sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+  sock.bind(('', port))
+  sock.listen(1)  # backlog of 1; meant to be used on localhost
+  log('Listening on port %d', port)
+
+  target.next()  # "prime" coroutine
+
+  while True:
+    client, addr = sock.accept()
+    log('Connection from %s', addr)
+
+    # The payload is a line right now.  Should we loop until we actually get a
+    # newline?  Could test a trickle.
+    line = client.recv(BUF_SIZE)
+    client.close()
+    if not line:
+      break
+
+    filename = line.strip()
+    try:
+      target.send(filename)
+    except StopIteration:
+      break
+
+
+def main(argv):
+  """Returns an exit code."""
+  port = None
+
+  # Just use simple getopt for now.  This isn't exposed to the UI really.
+  opts, argv = getopt.getopt(argv, 'p:')
+  for name, value in opts:
+    if name == '-p':
+      try:
+        port = int(value)
+      except ValueError:
+        raise Error('Invalid port %r' % value)
+    else:
+      raise AssertionError
+
+  # NOTE: This is the input base path.  We just join them with the filenames
+  # on stdin.
+  in_dir = argv[0]
+  out_dir = argv[1]
+
+  # PluginDispatchLoop is a coroutine.  It takes items to render on stdin.
+  loop = PluginDispatchLoop(in_dir, out_dir)
+
+  # TODO:
+  # create a tcp server.  reads one connection at a timv 
+
+  if port:
+    TcpServer(port, loop)
+  else:
+    Lines(sys.stdin, loop)
+
   return 0
 
 
 if __name__ == '__main__':
   try:
-    sys.exit(main(sys.argv))
+    sys.exit(main(sys.argv[1:]))
   except KeyboardInterrupt:
     log('Stopped')
     sys.exit(0)

webpipe/xrender_test.py

@@ -44,5 +44,22 @@ def testResources(self):
     print p
 
 
+def Echo():
+  while True:
+    filename = yield
+    if not filename:
+      break
+    print filename
+
+
+class TcpServerTest(unittest.TestCase):
+
+  def testServer(self):
+    e = Echo()
+    # This is a 'manual' test; it listens.
+    return
+    xrender.TcpServer(8002, e)
+
+
 if __name__ == '__main__':
   unittest.main()

wp-test.sh

@@ -3,6 +3,8 @@
 # Usage:
 #   ./wp-test.sh <function name>
 
+set -o nounset
+
 # TODO: make this a proper dep
 . ~/hg/taste/taste.sh