cco/loops
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

work in progress: http put tranport, + server-side integrator put handler

Browse source 
git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@1940 fd906abe-77d9-0310-91a1-e0d9ade77398
This commit is contained in:
helmutm 2007-08-20 17:03:09 +00:00
parent d564ff0de1
commit 993356b8ae
9 changed files with 161 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
agent/README.txt
View file

@ -48,8 +48,8 @@ This already provides all needed sections (transport, crawl, ui), so
we can directly put information into these sections by loading a
string with the corresponding assignment.
>>> config.load('transport.url = "http://loops.cy55.de"')
>>> config.transport.url
>>> config.load('transport.serverURL = "http://loops.cy55.de"')
>>> config.transport.serverURL
'http://loops.cy55.de'
This setting may also contain indexed access; thus we can model
@ -76,11 +76,11 @@ it with a default if not found, in one statement.
>>> config.ui.web.setdefault('port', 8080)
8081
>>> config.transport.setdefault('user', 'loops')
>>> config.transport.setdefault('userName', 'loops')
'loops'
>>> sorted(config.transport.items())
[('url', 'http://loops.cy55.de'), ('user', 'loops')]
[('serverURL', 'http://loops.cy55.de'), ('userName', 'loops')]
We can output a configuration in a form that is ready for loading
just by converting it to a string representation.
@ -88,8 +88,8 @@ just by converting it to a string representation.
>>> print config
crawl[0].directory = 'documents/projects'
crawl[0].type = 'filesystem'
transport.url = 'http://loops.cy55.de'
transport.user = 'loops'
transport.serverURL = 'http://loops.cy55.de'
transport.userName = 'loops'
ui.web.port = 8081
The configuration may also be saved to a file -
@ -108,8 +108,8 @@ for storage; normally it would be stored in the users home directory.
>>> print open(fn).read()
crawl[0].directory = 'documents/projects'
crawl[0].type = 'filesystem'
transport.url = 'http://loops.cy55.de'
transport.user = 'loops'
transport.serverURL = 'http://loops.cy55.de'
transport.userName = 'loops'
ui.web.port = 8081
Cleaning up up...
@ -185,7 +185,7 @@ Let's start with a fresh agent, directly supplying the configuration
... crawl[0].starttime = %s
... crawl[0].transport = 'dummy'
... crawl[0].repeat = 0
... transport.url = 'http://loops.cy55.de'
... transport.serverURL = 'http://loops.cy55.de'
... ''' % int(time())
>>> agent = core.Agent(config)
@ -261,12 +261,12 @@ Transport
Configuration
- ``transport.url``: URL of the target loops site, e.g.
- ``transport.serverURL``: URL of the target loops site, e.g.
"http://z3.loops.cy55.de/bwp/d5"
- ``transport.user``, ``transport.password`` for logging in to loops
- ``transport.machine: name under which the client computer is
- ``transport.userName``, ``transport.password`` for logging in to loops
- ``transport.machineName: name under which the client computer is
known to the loops server
- ``transport.method``, e.g. "put"
- ``transport.method``, e.g. "PUT"
The following information is intended for the default transfer
protocol/method HTTP PUT but probably also pertains to other protocols

14
agent/core.py
View file

@ -56,21 +56,23 @@ class Agent(object):
self.stopper.scheduler = self.scheduler
self.logger = Logger(self)
def scheduleJobsFromConfig(self):
def scheduleJobsFromConfig(self, stop=False):
config = self.config
scheduler = self.scheduler
lastJob = None
for idx, info in enumerate(config.crawl):
crawlType = info.type
factory = self.crawlTypes.get(crawlType)
if factory is not None:
job = factory()
job = lastJob = factory()
job.params = dict((name, value)
for name, value in info.items()
if name not in job.baseProperties)
transportType = info.transport or 'httpput'
factory = self.transportTypes.get(transportType)
if factory is not None:
transporter = factory(self)
params = dict(config.transport.items())
transporter = factory(self, **params)
# TODO: configure transporter or - better -
# set up transporter(s) just once
job.successors.append(transporter.createJob())
@ -79,4 +81,10 @@ class Agent(object):
# TODO: remove job from config
# TODO: put repeating info in config
# TODO: remember last run for repeating job
if stop:
if lastJob is not None:
lastTrJob = lastJob.successors[-1]
lastTrJob.successors.append(self.stopper)
else:
self.scheduler.schedule(self.stopper)

4
agent/schedule.py
View file

@ -83,6 +83,7 @@ class Job(object):
def run(self):
d = self.execute()
d.addCallback(self.finishRun)
d.addErrback(self.logError)
self.whenStarted(self)
# TODO: logging
@ -100,6 +101,9 @@ class Job(object):
if self.repeat:
self.reschedule(int(time() + self.repeat))
def logError(self, error):
print '*** error on running job:', error
def copy(self):
newJob = Job()
newJob.params = self.params

21
agent/testing/client.py
View file

@ -1,3 +1,4 @@
#! /usr/bin/env python
#
# Copyright (c) 2007 Helmut Merz helmutm@cy55.de
#
@ -27,6 +28,7 @@ $Id$
"""
import os
import sys
from time import time
from twisted.internet import reactor
@ -35,19 +37,14 @@ from loops.agent.crawl.filesystem import CrawlingJob
from loops.agent.transport.base import Transporter
from loops.agent.tests import baseDir
agent = Agent()
scheduler = agent.scheduler
if len(sys.argv) > 1:
cfgName = sys.argv[1]
else:
cfgName = os.path.join(baseDir, 'testing', 'testing.cfg')
cfg = open(cfgName)
dirname = os.path.join(baseDir, 'testing', 'data')
crawlJob = CrawlingJob(directory=dirname)
transporter = Transporter(agent)
transporter.serverURL = 'http://localhost:8123/loops'
transportJob = transporter.createJob()
crawlJob.successors.append(transportJob)
transportJob.successors.append(agent.stopper)
scheduler.schedule(crawlJob)
agent = Agent(cfg.read())
agent.scheduleJobsFromConfig(stop=True)
print 'Starting reactor.'

7
agent/testing/testing.cfg Normal file
View file

@ -0,0 +1,7 @@
crawl[0].type = 'filesystem'
crawl[0].directory = 'loops/agent/testing/data'
crawl[0].pattern = '*.txt'
crawl[0].transport = 'httpput'
crawl[0].repeat = 0
transport.serverURL = 'http://localhost:8123/loops'
#transport.serverURL = 'http://localhost:12080/sites/testsite/resources/'

21
agent/transport/base.py
View file

@ -48,13 +48,17 @@ class TransportJob(Job):
d = self.transporter.transfer(resource)
transfers.append(d)
d.addCallback(self.logTransfer)
d.addErrback(self.logError)
return DeferredList(transfers)
def logTransfer(self, result):
def logTransfer(self, result, err=None):
# TODO: logging
# self.transporter.agent.logger.log(...)
pass
def logError(self, error):
print '*** error on transfer', self.transporter.serverURL, error
class Transporter(object):
@ -68,8 +72,10 @@ class Transporter(object):
userName = 'nobody'
password = 'secret'
def __init__(self, agent):
def __init__(self, agent, **params):
self.agent = agent
for k, v in params.items():
setattr(self, k ,v)
def createJob(self):
return self.jobFactory(self)
@ -88,17 +94,20 @@ class Transporter(object):
deferreds = []
metadata = resource.metadata
if metadata is not None:
url = self.makePath('meta', app, path, 'xml')
url = self.makePath('.meta', app, path, 'xml')
deferreds.append(
getPage(url, method=self.method, postdata=metadata.asXML()))
url = self.makePath('data', app, path)
url = self.makePath('.data', app, path)
deferreds.append(getPage(url, method=self.method, postdata=text))
return DeferredList(deferreds)
return DeferredList(deferreds, fireOnOneErrback=True)
def makePath(self, infoType, app, path, extension=None):
if path.startswith('/'):
path = path[1:]
fullPath = '/'.join((self.serverURL, infoType,
url = self.serverURL
if url.endswith('/'):
url = url[:-1]
fullPath = '/'.join((url, infoType,
self.machineName, self.userName, app, path))
if extension:
fullPath += '.' + extension

22
integrator/README.txt
View file

@ -136,6 +136,28 @@ But if one of the referenced objects is not found any more it will be deleted.
('fullpath', {'subdirectory': '...zope'}, 'zope3.txt')
Uploading Resources with HTTP PUT Requests
==========================================
>>> from zope.publisher.browser import TestRequest
>>> from loops.integrator.put import ResourceManagerTraverser
>>> rrinfo = 'local/user/filesystem'
>>> rrpath = 'testing/data/file1.txt'
>>> rrid = '/'.join((rrinfo, rrpath))
>>> baseUrl = 'http://127.0.0.1/loops/resources'
>>> url = '/'.join((baseUrl, '.data', rrid))
>>> request = TestRequest(url)
>>> request.method = 'PUT'
>>> request._traversal_stack = list(reversed(rrid.split('/')))
>>> traverser = ResourceManagerTraverser(resources, request)
>>> traverser.publishTraverse(request, '.data')
*** resources.PUT .data testing,data,file1.txt local user filesystem
<...DummyWriteFile object ...>
Fin de partie
=============

24
integrator/configure.zcml
View file

@ -16,18 +16,28 @@
set_schema="loops.integrator.interfaces.IExternalCollection" />
</zope:class>
<zope:class class="loops.integrator.put.DummyWriteFile">
<allow attributes="write" />
</zope:class>
<zope:utility
factory="loops.integrator.collection.DirectoryCollectionProvider" />
<!-- view(s) -->
<zope:adapter
name="collection.html"
for="loops.interfaces.IConcept
zope.publisher.interfaces.browser.IBrowserRequest"
provides="zope.interface.Interface"
factory="loops.integrator.browser.ExternalCollectionView"
permission="zope.View"
/>
name="collection.html"
for="loops.interfaces.IConcept
zope.publisher.interfaces.browser.IBrowserRequest"
provides="zope.interface.Interface"
factory="loops.integrator.browser.ExternalCollectionView"
permission="zope.View" />
<zope:adapter
for="loops.interfaces.IResourceManager
zope.publisher.interfaces.http.IHTTPRequest"
factory="loops.integrator.put.ResourceManagerTraverser"
provides="zope.publisher.interfaces.IPublishTraverse"
permission="zope.Public" />
</configure>

63
integrator/put.py Normal file
View file

@ -0,0 +1,63 @@
#
# Copyright (c) 2007 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
"""
Traversal adapter for PUT requests, e.g. coming from loops.agent.
$Id$
"""
from zope import interface, component
from zope.interface import implements
from zope.component import adapts
from zope.app.container.traversal import ContainerTraverser, ItemTraverser
from zope.cachedescriptors.property import Lazy
from zope.filerepresentation.interfaces import IWriteFile
from zope.publisher.interfaces import IPublishTraverse
from loops.interfaces import IResourceManager
class ResourceManagerTraverser(ItemTraverser):
implements(IPublishTraverse)
adapts(IResourceManager)
def publishTraverse(self, request, name):
if request.method == 'PUT':
if name.startswith('.'):
stack = request._traversal_stack
#stack = request.getTraversalStack()
#print '*** resources.PUT', name, '/'.join(stack)
machine, user, app = stack.pop(), stack.pop(), stack.pop()
path = '/'.join(reversed(stack))
path = path.replace(',', ',,').replace('/', ',')
for i in range(len(stack)):
stack.pop()
print '*** resources.PUT', name, path, machine, user, app
return DummyWriteFile()
return super(ResourceManagerTraverser, self).publishTraverse(request, name)
class DummyWriteFile(object):
implements(IWriteFile)
def write(self, text):
print '*** content', repr(text)