diff --git a/agent/README.txt b/agent/README.txt
index 9b6596b..3778fb4 100644
--- a/agent/README.txt
+++ b/agent/README.txt
@@ -48,8 +48,8 @@ This already provides all needed sections (transport, crawl, ui), so
we can directly put information into these sections by loading a
string with the corresponding assignment.
- >>> config.load('transport.url = "http://loops.cy55.de"')
- >>> config.transport.url
+ >>> config.load('transport.serverURL = "http://loops.cy55.de"')
+ >>> config.transport.serverURL
'http://loops.cy55.de'
This setting may also contain indexed access; thus we can model
@@ -76,11 +76,11 @@ it with a default if not found, in one statement.
>>> config.ui.web.setdefault('port', 8080)
8081
- >>> config.transport.setdefault('user', 'loops')
+ >>> config.transport.setdefault('userName', 'loops')
'loops'
>>> sorted(config.transport.items())
- [('url', 'http://loops.cy55.de'), ('user', 'loops')]
+ [('serverURL', 'http://loops.cy55.de'), ('userName', 'loops')]
We can output a configuration in a form that is ready for loading
just by converting it to a string representation.
@@ -88,8 +88,8 @@ just by converting it to a string representation.
>>> print config
crawl[0].directory = 'documents/projects'
crawl[0].type = 'filesystem'
- transport.url = 'http://loops.cy55.de'
- transport.user = 'loops'
+ transport.serverURL = 'http://loops.cy55.de'
+ transport.userName = 'loops'
ui.web.port = 8081
The configuration may also be saved to a file -
@@ -108,8 +108,8 @@ for storage; normally it would be stored in the users home directory.
>>> print open(fn).read()
crawl[0].directory = 'documents/projects'
crawl[0].type = 'filesystem'
- transport.url = 'http://loops.cy55.de'
- transport.user = 'loops'
+ transport.serverURL = 'http://loops.cy55.de'
+ transport.userName = 'loops'
ui.web.port = 8081
Cleaning up up...
@@ -185,7 +185,7 @@ Let's start with a fresh agent, directly supplying the configuration
... crawl[0].starttime = %s
... crawl[0].transport = 'dummy'
... crawl[0].repeat = 0
- ... transport.url = 'http://loops.cy55.de'
+ ... transport.serverURL = 'http://loops.cy55.de'
... ''' % int(time())
>>> agent = core.Agent(config)
@@ -261,12 +261,12 @@ Transport
Configuration
-- ``transport.url``: URL of the target loops site, e.g.
+- ``transport.serverURL``: URL of the target loops site, e.g.
"http://z3.loops.cy55.de/bwp/d5"
-- ``transport.user``, ``transport.password`` for logging in to loops
-- ``transport.machine: name under which the client computer is
+- ``transport.userName``, ``transport.password`` for logging in to loops
+- ``transport.machineName: name under which the client computer is
known to the loops server
-- ``transport.method``, e.g. "put"
+- ``transport.method``, e.g. "PUT"
The following information is intended for the default transfer
protocol/method HTTP PUT but probably also pertains to other protocols
diff --git a/agent/core.py b/agent/core.py
index 9f3c531..95a7c75 100644
--- a/agent/core.py
+++ b/agent/core.py
@@ -56,21 +56,23 @@ class Agent(object):
self.stopper.scheduler = self.scheduler
self.logger = Logger(self)
- def scheduleJobsFromConfig(self):
+ def scheduleJobsFromConfig(self, stop=False):
config = self.config
scheduler = self.scheduler
+ lastJob = None
for idx, info in enumerate(config.crawl):
crawlType = info.type
factory = self.crawlTypes.get(crawlType)
if factory is not None:
- job = factory()
+ job = lastJob = factory()
job.params = dict((name, value)
for name, value in info.items()
if name not in job.baseProperties)
transportType = info.transport or 'httpput'
factory = self.transportTypes.get(transportType)
if factory is not None:
- transporter = factory(self)
+ params = dict(config.transport.items())
+ transporter = factory(self, **params)
# TODO: configure transporter or - better -
# set up transporter(s) just once
job.successors.append(transporter.createJob())
@@ -79,4 +81,10 @@ class Agent(object):
# TODO: remove job from config
# TODO: put repeating info in config
# TODO: remember last run for repeating job
+ if stop:
+ if lastJob is not None:
+ lastTrJob = lastJob.successors[-1]
+ lastTrJob.successors.append(self.stopper)
+ else:
+ self.scheduler.schedule(self.stopper)
diff --git a/agent/schedule.py b/agent/schedule.py
index 9219cbf..49794a1 100644
--- a/agent/schedule.py
+++ b/agent/schedule.py
@@ -83,6 +83,7 @@ class Job(object):
def run(self):
d = self.execute()
d.addCallback(self.finishRun)
+ d.addErrback(self.logError)
self.whenStarted(self)
# TODO: logging
@@ -100,6 +101,9 @@ class Job(object):
if self.repeat:
self.reschedule(int(time() + self.repeat))
+ def logError(self, error):
+ print '*** error on running job:', error
+
def copy(self):
newJob = Job()
newJob.params = self.params
diff --git a/agent/testing/client.py b/agent/testing/client.py
index 43e66b2..40ecff8 100644
--- a/agent/testing/client.py
+++ b/agent/testing/client.py
@@ -1,3 +1,4 @@
+#! /usr/bin/env python
#
# Copyright (c) 2007 Helmut Merz helmutm@cy55.de
#
@@ -27,6 +28,7 @@ $Id$
"""
import os
+import sys
from time import time
from twisted.internet import reactor
@@ -35,19 +37,14 @@ from loops.agent.crawl.filesystem import CrawlingJob
from loops.agent.transport.base import Transporter
from loops.agent.tests import baseDir
-agent = Agent()
-scheduler = agent.scheduler
+if len(sys.argv) > 1:
+ cfgName = sys.argv[1]
+else:
+ cfgName = os.path.join(baseDir, 'testing', 'testing.cfg')
+cfg = open(cfgName)
-dirname = os.path.join(baseDir, 'testing', 'data')
-crawlJob = CrawlingJob(directory=dirname)
-
-transporter = Transporter(agent)
-transporter.serverURL = 'http://localhost:8123/loops'
-transportJob = transporter.createJob()
-crawlJob.successors.append(transportJob)
-transportJob.successors.append(agent.stopper)
-
-scheduler.schedule(crawlJob)
+agent = Agent(cfg.read())
+agent.scheduleJobsFromConfig(stop=True)
print 'Starting reactor.'
diff --git a/agent/testing/testing.cfg b/agent/testing/testing.cfg
new file mode 100644
index 0000000..5a79029
--- /dev/null
+++ b/agent/testing/testing.cfg
@@ -0,0 +1,7 @@
+crawl[0].type = 'filesystem'
+crawl[0].directory = 'loops/agent/testing/data'
+crawl[0].pattern = '*.txt'
+crawl[0].transport = 'httpput'
+crawl[0].repeat = 0
+transport.serverURL = 'http://localhost:8123/loops'
+#transport.serverURL = 'http://localhost:12080/sites/testsite/resources/'
diff --git a/agent/transport/base.py b/agent/transport/base.py
index 3d82c43..c20e263 100644
--- a/agent/transport/base.py
+++ b/agent/transport/base.py
@@ -48,13 +48,17 @@ class TransportJob(Job):
d = self.transporter.transfer(resource)
transfers.append(d)
d.addCallback(self.logTransfer)
+ d.addErrback(self.logError)
return DeferredList(transfers)
- def logTransfer(self, result):
+ def logTransfer(self, result, err=None):
# TODO: logging
# self.transporter.agent.logger.log(...)
pass
+ def logError(self, error):
+ print '*** error on transfer', self.transporter.serverURL, error
+
class Transporter(object):
@@ -68,8 +72,10 @@ class Transporter(object):
userName = 'nobody'
password = 'secret'
- def __init__(self, agent):
+ def __init__(self, agent, **params):
self.agent = agent
+ for k, v in params.items():
+ setattr(self, k ,v)
def createJob(self):
return self.jobFactory(self)
@@ -88,17 +94,20 @@ class Transporter(object):
deferreds = []
metadata = resource.metadata
if metadata is not None:
- url = self.makePath('meta', app, path, 'xml')
+ url = self.makePath('.meta', app, path, 'xml')
deferreds.append(
getPage(url, method=self.method, postdata=metadata.asXML()))
- url = self.makePath('data', app, path)
+ url = self.makePath('.data', app, path)
deferreds.append(getPage(url, method=self.method, postdata=text))
- return DeferredList(deferreds)
+ return DeferredList(deferreds, fireOnOneErrback=True)
def makePath(self, infoType, app, path, extension=None):
if path.startswith('/'):
path = path[1:]
- fullPath = '/'.join((self.serverURL, infoType,
+ url = self.serverURL
+ if url.endswith('/'):
+ url = url[:-1]
+ fullPath = '/'.join((url, infoType,
self.machineName, self.userName, app, path))
if extension:
fullPath += '.' + extension
diff --git a/integrator/README.txt b/integrator/README.txt
index bd9cc40..11182b5 100644
--- a/integrator/README.txt
+++ b/integrator/README.txt
@@ -136,6 +136,28 @@ But if one of the referenced objects is not found any more it will be deleted.
('fullpath', {'subdirectory': '...zope'}, 'zope3.txt')
+Uploading Resources with HTTP PUT Requests
+==========================================
+
+ >>> from zope.publisher.browser import TestRequest
+ >>> from loops.integrator.put import ResourceManagerTraverser
+
+ >>> rrinfo = 'local/user/filesystem'
+ >>> rrpath = 'testing/data/file1.txt'
+ >>> rrid = '/'.join((rrinfo, rrpath))
+
+ >>> baseUrl = 'http://127.0.0.1/loops/resources'
+ >>> url = '/'.join((baseUrl, '.data', rrid))
+
+ >>> request = TestRequest(url)
+ >>> request.method = 'PUT'
+ >>> request._traversal_stack = list(reversed(rrid.split('/')))
+
+ >>> traverser = ResourceManagerTraverser(resources, request)
+ >>> traverser.publishTraverse(request, '.data')
+ *** resources.PUT .data testing,data,file1.txt local user filesystem
+ <...DummyWriteFile object ...>
+
Fin de partie
=============
diff --git a/integrator/configure.zcml b/integrator/configure.zcml
index 3f40343..031b320 100644
--- a/integrator/configure.zcml
+++ b/integrator/configure.zcml
@@ -16,18 +16,28 @@
set_schema="loops.integrator.interfaces.IExternalCollection" />
+
+
+
+
+ name="collection.html"
+ for="loops.interfaces.IConcept
+ zope.publisher.interfaces.browser.IBrowserRequest"
+ provides="zope.interface.Interface"
+ factory="loops.integrator.browser.ExternalCollectionView"
+ permission="zope.View" />
+
+
diff --git a/integrator/put.py b/integrator/put.py
new file mode 100644
index 0000000..76e521a
--- /dev/null
+++ b/integrator/put.py
@@ -0,0 +1,63 @@
+#
+# Copyright (c) 2007 Helmut Merz helmutm@cy55.de
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+
+"""
+Traversal adapter for PUT requests, e.g. coming from loops.agent.
+
+$Id$
+"""
+
+from zope import interface, component
+from zope.interface import implements
+from zope.component import adapts
+from zope.app.container.traversal import ContainerTraverser, ItemTraverser
+from zope.cachedescriptors.property import Lazy
+from zope.filerepresentation.interfaces import IWriteFile
+from zope.publisher.interfaces import IPublishTraverse
+
+from loops.interfaces import IResourceManager
+
+
+class ResourceManagerTraverser(ItemTraverser):
+
+ implements(IPublishTraverse)
+ adapts(IResourceManager)
+
+ def publishTraverse(self, request, name):
+ if request.method == 'PUT':
+ if name.startswith('.'):
+ stack = request._traversal_stack
+ #stack = request.getTraversalStack()
+ #print '*** resources.PUT', name, '/'.join(stack)
+ machine, user, app = stack.pop(), stack.pop(), stack.pop()
+ path = '/'.join(reversed(stack))
+ path = path.replace(',', ',,').replace('/', ',')
+ for i in range(len(stack)):
+ stack.pop()
+ print '*** resources.PUT', name, path, machine, user, app
+ return DummyWriteFile()
+ return super(ResourceManagerTraverser, self).publishTraverse(request, name)
+
+
+class DummyWriteFile(object):
+
+ implements(IWriteFile)
+
+ def write(self, text):
+ print '*** content', repr(text)
+