work in progress: http put tranport, + server-side integrator put handler

git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@1940 fd906abe-77d9-0310-91a1-e0d9ade77398

agent/README.txt

@@ -48,8 +48,8 @@ This already provides all needed sections (transport, crawl, ui), so
 we can directly put information into these sections by loading a
 string with the corresponding assignment.
 
-  >>> config.load('transport.url = "http://loops.cy55.de"')
+  >>> config.load('transport.serverURL = "http://loops.cy55.de"')
-  >>> config.transport.url
+  >>> config.transport.serverURL
   'http://loops.cy55.de'
 
 This setting may also contain indexed access; thus we can model
@@ -76,11 +76,11 @@ it with a default if not found, in one statement.
 
   >>> config.ui.web.setdefault('port', 8080)
   8081
-  >>> config.transport.setdefault('user', 'loops')
+  >>> config.transport.setdefault('userName', 'loops')
   'loops'
 
   >>> sorted(config.transport.items())
-  [('url', 'http://loops.cy55.de'), ('user', 'loops')]
+  [('serverURL', 'http://loops.cy55.de'), ('userName', 'loops')]
 
 We can output a configuration in a form that is ready for loading
 just by converting it to a string representation.
@@ -88,8 +88,8 @@ just by converting it to a string representation.
   >>> print config
   crawl[0].directory = 'documents/projects'
   crawl[0].type = 'filesystem'
-  transport.url = 'http://loops.cy55.de'
+  transport.serverURL = 'http://loops.cy55.de'
-  transport.user = 'loops'
+  transport.userName = 'loops'
   ui.web.port = 8081
 
 The configuration may also be saved to a file -
@@ -108,8 +108,8 @@ for storage; normally it would be stored in the users home directory.
   >>> print open(fn).read()
   crawl[0].directory = 'documents/projects'
   crawl[0].type = 'filesystem'
-  transport.url = 'http://loops.cy55.de'
+  transport.serverURL = 'http://loops.cy55.de'
-  transport.user = 'loops'
+  transport.userName = 'loops'
   ui.web.port = 8081
 
 Cleaning up up...
@@ -185,7 +185,7 @@ Let's start with a fresh agent, directly supplying the configuration
   ... crawl[0].starttime = %s
   ... crawl[0].transport = 'dummy'
   ... crawl[0].repeat = 0
-  ... transport.url = 'http://loops.cy55.de'
+  ... transport.serverURL = 'http://loops.cy55.de'
   ... ''' % int(time())
 
   >>> agent = core.Agent(config)
@@ -261,12 +261,12 @@ Transport
 
 Configuration
 
-- ``transport.url``: URL of the target loops site, e.g.
+- ``transport.serverURL``: URL of the target loops site, e.g.
   "http://z3.loops.cy55.de/bwp/d5"
-- ``transport.user``, ``transport.password`` for logging in to loops
+- ``transport.userName``, ``transport.password`` for logging in to loops
-- ``transport.machine: name under which the client computer is
+- ``transport.machineName: name under which the client computer is
   known to the loops server
-- ``transport.method``, e.g. "put"
+- ``transport.method``, e.g. "PUT"
 
 The following information is intended for the default transfer
 protocol/method HTTP PUT but probably also pertains to other protocols

agent/core.py

@@ -56,21 +56,23 @@ class Agent(object):
         self.stopper.scheduler = self.scheduler
         self.logger = Logger(self)
 
-    def scheduleJobsFromConfig(self):
+    def scheduleJobsFromConfig(self, stop=False):
         config = self.config
         scheduler = self.scheduler
+        lastJob = None
         for idx, info in enumerate(config.crawl):
             crawlType = info.type
             factory = self.crawlTypes.get(crawlType)
             if factory is not None:
-                job = factory()
+                job = lastJob = factory()
                 job.params = dict((name, value)
                                 for name, value in info.items()
                                 if name not in job.baseProperties)
                 transportType = info.transport or 'httpput'
                 factory = self.transportTypes.get(transportType)
                 if factory is not None:
-                    transporter = factory(self)
+                    params = dict(config.transport.items())
+                    transporter = factory(self, **params)
                     # TODO: configure transporter or - better -
                     #       set up transporter(s) just once
                     job.successors.append(transporter.createJob())
@@ -79,4 +81,10 @@ class Agent(object):
                 # TODO: remove job from config
                 # TODO: put repeating info in config
                 # TODO: remember last run for repeating job
+        if stop:
+            if lastJob is not None:
+                lastTrJob = lastJob.successors[-1]
+                lastTrJob.successors.append(self.stopper)
+            else:
+                self.scheduler.schedule(self.stopper)
 

agent/schedule.py

@@ -83,6 +83,7 @@ class Job(object):
     def run(self):
         d = self.execute()
         d.addCallback(self.finishRun)
+        d.addErrback(self.logError)
         self.whenStarted(self)
         # TODO: logging
 
@@ -100,6 +101,9 @@ class Job(object):
         if self.repeat:
             self.reschedule(int(time() + self.repeat))
 
+    def logError(self, error):
+        print '*** error on running job:', error
+
     def copy(self):
         newJob = Job()
         newJob.params = self.params

agent/testing/client.py

@@ -1,3 +1,4 @@
+#! /usr/bin/env python
 #
 #  Copyright (c) 2007 Helmut Merz helmutm@cy55.de
 #
@@ -27,6 +28,7 @@ $Id$
 """
 
 import os
+import sys
 from time import time
 from twisted.internet import reactor
 
@@ -35,19 +37,14 @@ from loops.agent.crawl.filesystem import CrawlingJob
 from loops.agent.transport.base import Transporter
 from loops.agent.tests import baseDir
 
-agent = Agent()
+if len(sys.argv) > 1:
-scheduler = agent.scheduler
+    cfgName = sys.argv[1]
+else:
+    cfgName = os.path.join(baseDir, 'testing', 'testing.cfg')
+cfg = open(cfgName)
 
-dirname = os.path.join(baseDir, 'testing', 'data')
+agent = Agent(cfg.read())
-crawlJob = CrawlingJob(directory=dirname)
+agent.scheduleJobsFromConfig(stop=True)
-
-transporter = Transporter(agent)
-transporter.serverURL = 'http://localhost:8123/loops'
-transportJob = transporter.createJob()
-crawlJob.successors.append(transportJob)
-transportJob.successors.append(agent.stopper)
-
-scheduler.schedule(crawlJob)
 
 print 'Starting reactor.'
 

agent/testing/testing.cfg

@@ -0,0 +1,7 @@
+crawl[0].type = 'filesystem'
+crawl[0].directory = 'loops/agent/testing/data'
+crawl[0].pattern = '*.txt'
+crawl[0].transport = 'httpput'
+crawl[0].repeat = 0
+transport.serverURL = 'http://localhost:8123/loops'
+#transport.serverURL = 'http://localhost:12080/sites/testsite/resources/'

agent/transport/base.py

@@ -48,13 +48,17 @@ class TransportJob(Job):
             d = self.transporter.transfer(resource)
             transfers.append(d)
             d.addCallback(self.logTransfer)
+            d.addErrback(self.logError)
         return DeferredList(transfers)
 
-    def logTransfer(self, result):
+    def logTransfer(self, result, err=None):
         # TODO: logging
         # self.transporter.agent.logger.log(...)
         pass
 
+    def logError(self, error):
+        print '*** error on transfer', self.transporter.serverURL, error
+
 
 class Transporter(object):
 
@@ -68,8 +72,10 @@ class Transporter(object):
     userName = 'nobody'
     password = 'secret'
 
-    def __init__(self, agent):
+    def __init__(self, agent, **params):
         self.agent = agent
+        for k, v in params.items():
+            setattr(self, k ,v)
 
     def createJob(self):
         return self.jobFactory(self)
@@ -88,17 +94,20 @@ class Transporter(object):
         deferreds = []
         metadata = resource.metadata
         if metadata is not None:
-            url = self.makePath('meta', app, path, 'xml')
+            url = self.makePath('.meta', app, path, 'xml')
             deferreds.append(
                     getPage(url, method=self.method, postdata=metadata.asXML()))
-        url = self.makePath('data', app, path)
+        url = self.makePath('.data', app, path)
         deferreds.append(getPage(url, method=self.method, postdata=text))
-        return DeferredList(deferreds)
+        return DeferredList(deferreds, fireOnOneErrback=True)
 
     def makePath(self, infoType, app, path, extension=None):
         if path.startswith('/'):
             path = path[1:]
-        fullPath = '/'.join((self.serverURL, infoType,
+        url = self.serverURL
+        if url.endswith('/'):
+            url = url[:-1]
+        fullPath = '/'.join((url, infoType,
                              self.machineName, self.userName, app, path))
         if extension:
             fullPath += '.' + extension

integrator/README.txt

@@ -136,6 +136,28 @@ But if one of the referenced objects is not found any more it will be deleted.
   ('fullpath', {'subdirectory': '...zope'}, 'zope3.txt')
 
 
+Uploading Resources with HTTP PUT Requests
+==========================================
+
+  >>> from zope.publisher.browser import TestRequest
+  >>> from loops.integrator.put import ResourceManagerTraverser
+
+  >>> rrinfo = 'local/user/filesystem'
+  >>> rrpath = 'testing/data/file1.txt'
+  >>> rrid = '/'.join((rrinfo, rrpath))
+
+  >>> baseUrl = 'http://127.0.0.1/loops/resources'
+  >>> url = '/'.join((baseUrl, '.data', rrid))
+
+  >>> request = TestRequest(url)
+  >>> request.method = 'PUT'
+  >>> request._traversal_stack = list(reversed(rrid.split('/')))
+
+  >>> traverser = ResourceManagerTraverser(resources, request)
+  >>> traverser.publishTraverse(request, '.data')
+  *** resources.PUT .data testing,data,file1.txt local user filesystem
+  <...DummyWriteFile object ...>
+
 Fin de partie
 =============
 

integrator/configure.zcml

@@ -16,18 +16,28 @@
              set_schema="loops.integrator.interfaces.IExternalCollection" />
   </zope:class>
 
+  <zope:class class="loops.integrator.put.DummyWriteFile">
+    <allow attributes="write" />
+  </zope:class>
+
   <zope:utility
       factory="loops.integrator.collection.DirectoryCollectionProvider" />
 
   <!-- view(s) -->
 
   <zope:adapter
-      name="collection.html"
+        name="collection.html"
-      for="loops.interfaces.IConcept
+        for="loops.interfaces.IConcept
-           zope.publisher.interfaces.browser.IBrowserRequest"
+             zope.publisher.interfaces.browser.IBrowserRequest"
-      provides="zope.interface.Interface"
+        provides="zope.interface.Interface"
-      factory="loops.integrator.browser.ExternalCollectionView"
+        factory="loops.integrator.browser.ExternalCollectionView"
-      permission="zope.View"
+        permission="zope.View" />
-      />
+
+  <zope:adapter
+        for="loops.interfaces.IResourceManager
+             zope.publisher.interfaces.http.IHTTPRequest"
+        factory="loops.integrator.put.ResourceManagerTraverser"
+        provides="zope.publisher.interfaces.IPublishTraverse"
+        permission="zope.Public" />
 
 </configure>

integrator/put.py

@@ -0,0 +1,63 @@
+#
+#  Copyright (c) 2007 Helmut Merz helmutm@cy55.de
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+
+"""
+Traversal adapter for PUT requests, e.g. coming from loops.agent.
+
+$Id$
+"""
+
+from zope import interface, component
+from zope.interface import implements
+from zope.component import adapts
+from zope.app.container.traversal import ContainerTraverser, ItemTraverser
+from zope.cachedescriptors.property import Lazy
+from zope.filerepresentation.interfaces import IWriteFile
+from zope.publisher.interfaces import IPublishTraverse
+
+from loops.interfaces import IResourceManager
+
+
+class ResourceManagerTraverser(ItemTraverser):
+
+    implements(IPublishTraverse)
+    adapts(IResourceManager)
+
+    def publishTraverse(self, request, name):
+        if request.method == 'PUT':
+            if name.startswith('.'):
+                stack = request._traversal_stack
+                #stack = request.getTraversalStack()
+                #print '*** resources.PUT', name, '/'.join(stack)
+                machine, user, app = stack.pop(), stack.pop(), stack.pop()
+                path = '/'.join(reversed(stack))
+                path = path.replace(',', ',,').replace('/', ',')
+                for i in range(len(stack)):
+                    stack.pop()
+                print '*** resources.PUT', name, path, machine, user, app
+                return DummyWriteFile()
+        return super(ResourceManagerTraverser, self).publishTraverse(request, name)
+
+
+class DummyWriteFile(object):
+
+    implements(IWriteFile)
+
+    def write(self, text):
+        print '*** content', repr(text)
+