diff --git a/agent/crawl/base.py b/agent/crawl/base.py index ca66266..dedad18 100644 --- a/agent/crawl/base.py +++ b/agent/crawl/base.py @@ -27,7 +27,7 @@ from zope.interface import implements from cybertools.agent.base.agent import Master from cybertools.agent.core.agent import QueueableAgent from cybertools.agent.interfaces import ICrawler -from cybertools.agent.interfaces import IResource +from cybertools.agent.interfaces import IResource, IMetadataSet from cybertools.agent.components import agents from twisted.internet.defer import succeed @@ -35,7 +35,7 @@ from twisted.internet.defer import succeed class Crawler(QueueableAgent): implements(ICrawler) - + def __init__(self, master, params={}): super(Crawler, self).__init__(master) @@ -54,6 +54,8 @@ class SampleCrawler(Crawler): d = succeed([]) return d +agents.register(SampleCrawler, Master, name='crawl.sample') + class Resource(object): @@ -71,6 +73,18 @@ class Resource(object): self.metadata = metadata +class Metadata(dict): -agents.register(SampleCrawler, Master, name='crawl.sample') + implements(IMetadataSet) + + def __init__(self, data=dict()): + for k in data: + self[k] = data[k] + + def asXML(self): + # TODO... + return '' + + def set(self, key, value): + self['key'] = value diff --git a/agent/crawl/filesystem.py b/agent/crawl/filesystem.py new file mode 100644 index 0000000..690486f --- /dev/null +++ b/agent/crawl/filesystem.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2008 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +Filesystem crawler. + +$Id$ +""" + +import os +from fnmatch import filter +from datetime import datetime +from twisted.internet.defer import Deferred +from zope.interface import implements + +from cybertools.agent.base.agent import Master +from cybertools.agent.components import agents +from cybertools.agent.crawl.base import Resource, Metadata +from cybertools.agent.crawl.base import Crawler +from cybertools.agent.util.task import coiterate + + + +class FilesystemCrawler(Crawler): + + def collect(self): + self.collected = [] + coiterate(self.crawlFilesystem()).addCallback(self.finished) + # TODO: addErrback() + self.deferred = Deferred() + return self.deferred + + def finished(self, result): + self.deferred.callback(self.collected) + + def crawlFilesystem(self): + directory = self.params.get('directory') + pattern = self.params.get('pattern') or '*' + lastRun = self.params.get('lastrun') or datetime(1980, 1, 1) + for path, dirs, files in os.walk(directory): + if '.svn' in dirs: + del dirs[dirs.index('.svn')] + for x in self.loadFiles(path, files, pattern, lastRun): + yield None + + def loadFiles(self, path, files, pattern, lastRun): + for f in filter(files, pattern): + filename = os.path.join(path, f) + mtime = datetime.fromtimestamp(os.path.getmtime(filename)) + if mtime <= lastRun: # file not changed + continue + meta = dict( + path=filename, + ) + self.collected.append(FileResource(filename, Metadata(meta))) + yield None + +agents.register(FilesystemCrawler, Master, name='crawl.filesystem') + + +class FileResource(Resource): + + def __init__(self, path, metadata=None): + self.path = path + self.metadata = metadata + + application = 'filesystem' + + @property + def data(self): + return open(self.path, 'r') + diff --git a/agent/crawl/filesystem.txt b/agent/crawl/filesystem.txt new file mode 100644 index 0000000..39a0ba0 --- /dev/null +++ b/agent/crawl/filesystem.txt @@ -0,0 +1,38 @@ +================================================ +Agents for Job Execution and Communication Tasks +================================================ + + ($Id$) + + >>> import os + >>> from time import time + + >>> from cybertools.agent.tests import tester, baseDir + >>> config = ''' + ... controller(names=['core.sample']) + ... scheduler(name='core') + ... logger(name='default', standard=30) + ... ''' + >>> from cybertools.agent.main import setup + >>> master = setup(config) + Starting agent application... + Using controllers core.sample. + + +Filesystem Crawler +================== + + >>> controller = master.controllers[0] + >>> controller.createAgent('crawl.filesystem', 'sample03') + +In the next step we request the start of a job, again via the controller. + + >>> path = os.path.join(baseDir, 'testing', 'data') + >>> controller.enterJob('sample', 'sample03', params=dict(directory=path)) + +The job is not executed immediately - we have to hand over control to +the twisted reactor first. + + >>> from cybertools.agent.tests import tester + >>> tester.iterate() + Job 00001 completed; result: [..., ...]; \ No newline at end of file diff --git a/agent/main.py b/agent/main.py index e6f520f..c0673cb 100755 --- a/agent/main.py +++ b/agent/main.py @@ -58,7 +58,7 @@ def setupEnvironment(config): from cybertools.agent.control import cmdline from cybertools.agent.system.windows import api api.setup(config) - from cybertools.agent.crawl import base, outlook + from cybertools.agent.crawl import base, filesystem, outlook def startReactor(): diff --git a/agent/testing/data/file1.txt b/agent/testing/data/file1.txt new file mode 100644 index 0000000..02c267f --- /dev/null +++ b/agent/testing/data/file1.txt @@ -0,0 +1 @@ +Data from file1.txt \ No newline at end of file diff --git a/agent/testing/data/subdir/file2.txt b/agent/testing/data/subdir/file2.txt new file mode 100644 index 0000000..493d31b --- /dev/null +++ b/agent/testing/data/subdir/file2.txt @@ -0,0 +1 @@ +Data from file2.txt \ No newline at end of file diff --git a/agent/tests.py b/agent/tests.py index f7323f0..7ecdc52 100755 --- a/agent/tests.py +++ b/agent/tests.py @@ -43,6 +43,7 @@ def test_suite(): unittest.makeSuite(Test), DocFileSuite('README.txt', optionflags=flags), DocFileSuite('crawl/README.txt', optionflags=flags), + DocFileSuite('crawl/filesystem.txt', optionflags=flags), DocFileSuite('crawl/outlook.txt', optionflags=flags), )) return testSuite