From f2e5ba51b97abd1e7b7ede64735308d7a6fe0a6f Mon Sep 17 00:00:00 2001 From: helmutm Date: Thu, 9 Aug 2007 17:51:17 +0000 Subject: [PATCH] move metadata into Resource class; more on httpput transport git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@1897 fd906abe-77d9-0310-91a1-e0d9ade77398 --- agent/crawl/base.py | 7 ++++--- agent/crawl/filesystem.py | 34 +++++++++++++++++----------------- agent/interfaces.py | 13 +++++++------ agent/testing/crawl.py | 3 ++- agent/testing/transport.py | 10 ++++++---- agent/tests.py | 1 + agent/transport/base.py | 5 +++-- agent/transport/httpput.py | 29 ++++++++++++++++------------- agent/transport/httpput.txt | 15 +++++++++++++++ 9 files changed, 71 insertions(+), 46 deletions(-) create mode 100644 agent/transport/httpput.txt diff --git a/agent/crawl/base.py b/agent/crawl/base.py index 0902484..59417cb 100644 --- a/agent/crawl/base.py +++ b/agent/crawl/base.py @@ -42,17 +42,18 @@ class CrawlingJob(Job): return self.collect() -class Metadata(object): +class Metadata(dict): implements(IMetadataSet) def __init__(self, data=dict()): - self.data = data + for k in data: + self[k] = data[k] def asXml(self): # TODO... return '' def set(self, key, value): - self.data['key'] = value + self['key'] = value diff --git a/agent/crawl/filesystem.py b/agent/crawl/filesystem.py index c1dcab6..bfc804d 100644 --- a/agent/crawl/filesystem.py +++ b/agent/crawl/filesystem.py @@ -22,7 +22,8 @@ Filesystem crawler. $Id$ """ -import os, re, stat +import os +from fnmatch import filter from datetime import datetime from twisted.internet.defer import Deferred from twisted.internet.task import coiterate @@ -46,32 +47,31 @@ class CrawlingJob(BaseCrawlingJob): self.deferred.callback(self.collected) def crawlFilesystem(self): - criteria = self.params - directory = criteria.get('directory') - pattern = re.compile(criteria.get('pattern') or '.*') + directory = self.params.get('directory') + pattern = self.params.get('pattern') or '*' + lastRun = self.params.get('lastrun') or datetime(1980, 1, 1) for path, dirs, files in os.walk(directory): if '.svn' in dirs: del dirs[dirs.index('.svn')] - for f in files: - if pattern.match(f): - filename = os.path.join(path, f) - mtime = datetime.fromtimestamp( - os.stat(filename)[stat.ST_MTIME]) - # TODO: check modification time - meta = dict( - path=filename, - ) - self.collected.append((FileResource(filename), - Metadata(meta))) - yield None + for f in filter(files, pattern): + filename = os.path.join(path, f) + mtime = datetime.fromtimestamp(os.path.getmtime(filename)) + if mtime <= lastRun: # file not changed + continue + meta = dict( + path=filename, + ) + self.collected.append(FileResource(filename, Metadata(meta))) + yield None class FileResource(object): implements(IResource) - def __init__(self, path): + def __init__(self, path, metadata=None): self.path = path + self.metadata = metadata @property def data(self): diff --git a/agent/interfaces.py b/agent/interfaces.py index 44798bf..bbabd5b 100644 --- a/agent/interfaces.py +++ b/agent/interfaces.py @@ -112,9 +112,12 @@ class IResource(Interface): data = Attribute("A string, file, or similar representation of the " "resource's content") + metadata = Attribute('Information describing this resource; ' + 'should be an IMetadataSet object.') + class IMetadataSet(Interface): - """ Metadata associated with a resource. + """ Metadata associated with a resource; sort of a mapping. """ def asXML(): @@ -145,11 +148,9 @@ class ITransporter(Interface): userName = Attribute('User name for logging in to the server.') password = Attribute('Password for logging in to the server.') - def transfer(resource, metadata=None): - """ Transfer the resource (typically just a file that may - be read) to the server. - - The resource may be associated with a metadata set. + def transfer(resource): + """ Transfer the resource (an object providing IResource) + to the server. """ diff --git a/agent/testing/crawl.py b/agent/testing/crawl.py index 52929a8..0452eef 100644 --- a/agent/testing/crawl.py +++ b/agent/testing/crawl.py @@ -39,7 +39,7 @@ class CrawlingJob(BaseCrawlingJob): return deferred def dataAvailable(self): - self.deferred.callback([(DummyResource(), None)]) + self.deferred.callback([DummyResource()]) class DummyResource(object): @@ -47,3 +47,4 @@ class DummyResource(object): implements(IResource) data = 'Dummy resource data for testing purposes.' + metadata = None diff --git a/agent/testing/transport.py b/agent/testing/transport.py index d755546..47274cd 100644 --- a/agent/testing/transport.py +++ b/agent/testing/transport.py @@ -40,8 +40,8 @@ class TransportJob(BaseJob): if result is None: print 'No data available.' else: - for res, meta in result: - d = self.transporter.transfer(res.data, meta) + for resource in result: + d = self.transporter.transfer(resource) return Deferred() @@ -49,14 +49,16 @@ class Transporter(BaseTransporter): jobFactory = TransportJob - def transfer(self, data, metadata=None): + def transfer(self, resource): + data = resource.data if type(data) is file: text = data.read() data.close() else: text = data + metadata = resource.metadata if metadata is not None: - print 'Metadata:', metadata.data + print 'Metadata:', metadata print 'Transferring:', text return Deferred() diff --git a/agent/tests.py b/agent/tests.py index 296d9be..d6f0dd9 100755 --- a/agent/tests.py +++ b/agent/tests.py @@ -62,6 +62,7 @@ def test_suite(): #standard_unittest.makeSuite(Test), doctest.DocFileSuite('README.txt', optionflags=flags), doctest.DocFileSuite('crawl/filesystem.txt', optionflags=flags), + doctest.DocFileSuite('transport/httpput.txt', optionflags=flags), )) if __name__ == '__main__': diff --git a/agent/transport/base.py b/agent/transport/base.py index fcd86d7..ed30d60 100644 --- a/agent/transport/base.py +++ b/agent/transport/base.py @@ -48,9 +48,10 @@ class Transporter(object): def __init__(self, agent): self.agent = agent - def transfer(self, data, metadata=None): + def transfer(self, resource): + data = resource.data if type(data) is file: - data = text.read() + text = data.read() data.close() else: text = data diff --git a/agent/transport/httpput.py b/agent/transport/httpput.py index 52a3306..fe3cf71 100644 --- a/agent/transport/httpput.py +++ b/agent/transport/httpput.py @@ -23,7 +23,7 @@ $Id$ """ from twisted.internet import reactor -from twisted.internet.defer import Deferred +from twisted.web.client import getPage from zope.interface import implements from loops.agent.interfaces import ITransporter, ITransportJob @@ -38,13 +38,13 @@ class TransportJob(Job): super(TransportJob, self).__init__() self.transporter = transporter - def execute(self, **kw): + def execute(self): result = kw.get('result') if result is None: print 'No data available.' else: - for r in result: - d = self.transporter.transfer(r[0].data, r[1], str) + for resource, metadata in result: + d = self.transporter.transfer(resource.data, metadata) return Deferred() @@ -62,15 +62,18 @@ class Transporter(object): def __init__(self, agent): self.agent = agent - config = agent.config + conf = agent.config + # TODO: get settings from conf - def transfer(self, resource, metadata=None, resourceType=file): - if resourceType is file: - data = resource.read() + def transfer(self, resource): + data = resource.data + if type(data) is file: + text = resource.read() resource.close() - elif resourceType is str: - data = resource - print 'Transferring:', data - return Deferred() - + else: + text = data + metadata = resource.metadata + url = self.serverURL + self.makePath(metadata) + d = getPage(url, method='PUT', postData=text) + return d diff --git a/agent/transport/httpput.txt b/agent/transport/httpput.txt new file mode 100644 index 0000000..343220a --- /dev/null +++ b/agent/transport/httpput.txt @@ -0,0 +1,15 @@ +====================================================== +loops.agent.transport.httpput - The HTTP PUT Transport +====================================================== + + ($Id$) + + >>> from time import time + + >>> from loops.agent.core import Agent + >>> from loops.agent.transport.httpput import Transporter, TransportJob + + >>> agent = Agent() + >>> transporter = Transporter(agent) + >>> job = TransportJob(transporter) +