From c47cc6d9b15ad36aedd1e7f70d9c7aa4b0d76435 Mon Sep 17 00:00:00 2001 From: helmutm Date: Sun, 23 Mar 2008 21:37:28 +0000 Subject: [PATCH] added export/import of resources and resource relations, storing resource data (text/content) in files git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@2469 fd906abe-77d9-0310-91a1-e0d9ade77398 --- external/README.txt | 38 ++++++++++++++--- external/base.py | 32 ++++++++++---- external/browser.py | 27 ++++++++++-- external/element.py | 67 ++++++++++++++++++++---------- external/testdata/import/doc04.txt | 1 + external/tests.py | 4 ++ setup.py | 27 ++++++++++++ util.py | 13 ++++++ 8 files changed, 171 insertions(+), 38 deletions(-) create mode 100644 external/testdata/import/doc04.txt diff --git a/external/README.txt b/external/README.txt index 280d598..37a51af 100644 --- a/external/README.txt +++ b/external/README.txt @@ -48,6 +48,22 @@ Creating the corresponding objects >>> adapted(concepts['myquery']).viewName 'mystuff.html' +Working with resources +---------------------- + + >>> import os + >>> from loops.external.tests import dataDirectory + >>> loader.resourceDirectory = os.path.join(dataDirectory, 'import') + + >>> input = ("resource('doc04.txt', u'Document 4', 'textdocument')\n" + ... "resourceRelation('myquery', 'doc04.txt', 'standard')") + >>> reader = PyReader() + >>> elements = reader.read(input) + >>> loader.load(elements) + + >>> sorted(resources) + [u'd001.txt', u'd002.txt', u'd003.txt', u'doc04.txt'] + Working with nodes ------------------ @@ -66,11 +82,10 @@ Extracting elements ------------------- >>> from loops.external.base import Extractor - - >>> extractor = Extractor(loopsRoot) + >>> extractor = Extractor(loopsRoot, os.path.join(dataDirectory, 'export')) >>> elements = list(extractor.extract()) >>> len(elements) - 15 + 20 Writing object information to the external storage -------------------------------------------------- @@ -86,7 +101,9 @@ Writing object information to the external storage type(u'query', u'Query', options=u'', typeInterface='loops.query.IQueryConcept', viewName=u'')... concept(u'myquery', u'My Query', u'query', options=u'', viewName='mystuff.html')... - child(u'projects', u'customer', u'standard') + child(u'projects', u'customer', u'standard')... + resource(u'doc04.txt', u'Document 4', u'textdocument', contentType='text/restructured') + resourceRelation(u'myquery', u'doc04.txt', u'standard') node('home', u'Home', '', u'menu', body=u'Welcome') node('myquery', u'My Query', 'home', u'page', target=u'concepts/myquery')... @@ -97,7 +114,18 @@ The Export/Import View >>> from loops.external.browser import ExportImport >>> from zope.publisher.browser import TestRequest - >>> input = {'field.data': output} + >>> input = {'field.data': output, 'resourceDirectory': dataDirectory} >>> view = ExportImport(loopsRoot, TestRequest(input)) >>> view.upload() False + + +Fin de Partie +============= + + >>> placefulTearDown() + + >>> exportDir = os.path.join(dataDirectory, 'export') + >>> for fname in os.listdir(exportDir): + ... os.unlink(os.path.join(exportDir, fname)) + diff --git a/external/base.py b/external/base.py index ffb01e3..4effaf3 100644 --- a/external/base.py +++ b/external/base.py @@ -25,6 +25,8 @@ $Id$ from cStringIO import StringIO import itertools +import os +import zdaemon from zope import component from zope.cachedescriptors.property import Lazy from zope.interface import implements @@ -42,8 +44,9 @@ from loops.setup import SetupManager class Base(object): - def __init__(self, context): + def __init__(self, context, resourceDirectory=None): self.context = context + self.resourceDirectory = resourceDirectory @Lazy def concepts(self): @@ -70,8 +73,8 @@ class Loader(Base, SetupManager): implements(ILoader) - def __init__(self, context): - self.context = context + def __init__(self, context, resourceDirectory=None): + super(Loader, self).__init__(context, resourceDirectory) self.logger = StringIO() def load(self, elements): @@ -90,8 +93,8 @@ class Extractor(Base): return itertools.chain(self.extractTypes(), self.extractConcepts(), self.extractChildren(), - #self.extractResources(), - #self.extractResourceRelations(), + self.extractResources(), + self.extractResourceRelations(), self.extractNodes(), ) @@ -111,12 +114,14 @@ class Extractor(Base): yield conceptElement(name, obj.title, tp, **data) def extractResources(self): - resourceElement = elementTypes['resource'] + elementClass = elementTypes['resource'] for name, obj in self.resources.items(): # TODO: handle ``data`` attribute... data = self.getObjectData(obj) tp = getName(obj.resourceType) - yield resourceElement(name, obj.title, tp, **data) + element = elementClass(name, obj.title, tp, **data) + element.processExport(self) + yield element def getObjectData(self, obj): aObj = adapted(obj) @@ -149,6 +154,19 @@ class Extractor(Base): args.append(r.relevance) yield childElement(*args) + def extractResourceRelations(self): + elementClass = elementTypes['resourceRelation'] + typePredicate = self.typePredicate + for c in self.concepts.values(): + for r in c.getResourceRelations(): + if r.predicate != typePredicate: + args = [getName(r.first), getName(r.second), getName(r.predicate)] + if r.order != 0: + args.append(r.order) + if r.relevance != 1.0: + args.append(r.relevance) + yield elementClass(*args) + def extractNodes(self, parent=None, path=''): if parent is None: parent = self.views diff --git a/external/browser.py b/external/browser.py index f0c8343..9e242f0 100644 --- a/external/browser.py +++ b/external/browser.py @@ -22,16 +22,19 @@ view class(es) for import/export. $Id$ """ +from cStringIO import StringIO +import os from zope import component from zope.interface import Interface, implements from zope.app import zapi from zope.app.pagetemplate.viewpagetemplatefile import ViewPageTemplateFile from zope.cachedescriptors.property import Lazy from zope.security.proxy import removeSecurityProxy -from cStringIO import StringIO +from zope.traversing.api import getPath from loops.external.base import Loader, Extractor from loops.external.interfaces import IReader, IWriter +from loops import util class ExportImport(object): @@ -43,6 +46,22 @@ class ExportImport(object): self.request = request self.message = u'' + @Lazy + def baseDirectory(self): + return util.getVarDirectory(self.request) + + @Lazy + def sitePath(self): + return getPath(self.context)[1:].replace('/', '_') + + @Lazy + def resourceImportDirectory(self): + return os.path.join(self.baseDirectory, 'import', self.sitePath) + + @Lazy + def resourceExportDirectory(self): + return os.path.join(self.baseDirectory, 'export', self.sitePath) + def submit(self): action = self.request.get('loops.action', None) if action: @@ -53,7 +72,7 @@ class ExportImport(object): def export(self): f = StringIO() - extractor = Extractor(self.context) + extractor = Extractor(self.context, self.resourceExportDirectory) elements = extractor.extract() writer = component.getUtility(IWriter) writer.write(elements, f) @@ -64,11 +83,13 @@ class ExportImport(object): def upload(self): data = self.request.get('field.data', None) + resourceImportDirectory = (self.request.get('resourceImportDirectory', None) + or self.resourceImportDirectory) if not data: return False reader = component.getUtility(IReader) elements = reader.read(data) - loader = Loader(self.context) + loader = Loader(self.context, ) loader.load(elements) self.message = u'Content uploaded and imported.' return False diff --git a/external/element.py b/external/element.py index afbc3f5..bbde812 100644 --- a/external/element.py +++ b/external/element.py @@ -23,6 +23,7 @@ and import of loops objects. $Id$ """ +import os from zope.cachedescriptors.property import Lazy from zope.dottedname.resolve import resolve from zope.interface import Interface, implements @@ -45,6 +46,9 @@ class Element(dict): for k, v in kw.items(): self[k] = v + def processExport(self, extractor): + pass + def __call__(self, loader): pass @@ -95,6 +99,44 @@ class ChildElement(Element): loader.assignChild(self['first'], self['second'], self['predicate']) +class ResourceElement(Element): + + elementType = 'resource' + posArgs = ('name', 'title', 'type') + + def processExport(self, extractor): + content = self.pop('data', '') + if (self.get('contentType', '').startswith('text/') + and isinstance(content, unicode)): + content = content.encode('UTF-8') + dataPath = os.path.join(extractor.resourceDirectory, self['name']) + f = open(dataPath, 'w') + f.write(content) + f.close() + + def __call__(self, loader): + type = loader.concepts[self['type']] + kw = dict((k, v) for k, v in self.items() + if k not in self.posArgs) + dataPath = os.path.join(loader.resourceDirectory, self['name']) + if os.path.exists(dataPath): + f = open(dataPath, 'r') + content = f.read() + if self.get('contentType', '').startswith('text/'): + content = content.decode('UTF-8') + kw['data'] = content + f.close() + loader.addResource(self['name'], self['title'], type, **kw) + + +class ResourceRelationElement(ChildElement): + + elementType = 'resourceRelation' + + def __call__(self, loader): + loader.assignResource(self['first'], self['second'], self['predicate']) + + class NodeElement(Element): elementType = 'node' @@ -118,34 +160,13 @@ class NodeElement(Element): node.target = targetObject -# not yet implemented - -class ResourceElement(Element): - - elementType = 'resource' - posArgs = ('name', 'title', 'type') - - def __call__(self, loader): - type = loader.concepts[self['type']] - kw = dict((k, v) for k, v in self.items() - if k not in self.posArgs) - loader.addResource(self['name'], self['title'], type, **kw) - - -class ResourceRelationElement(ChildElement): - - elementType = 'resourceRelation' - - def __call__(self, loader): - loader.assignResource(self['first'], self['second'], self['predicate']) - - # element registry elementTypes = dict( type=TypeElement, concept=ConceptElement, - resource=ResourceElement, child=ChildElement, + resource=ResourceElement, + resourceRelation=ResourceRelationElement, node=NodeElement, ) diff --git a/external/testdata/import/doc04.txt b/external/testdata/import/doc04.txt new file mode 100644 index 0000000..00a6d36 --- /dev/null +++ b/external/testdata/import/doc04.txt @@ -0,0 +1 @@ +Test File Contents \ No newline at end of file diff --git a/external/tests.py b/external/tests.py index ab4e262..3f50aac 100755 --- a/external/tests.py +++ b/external/tests.py @@ -1,10 +1,14 @@ # $Id$ import unittest, doctest +import os from zope.testing.doctestunit import DocFileSuite from zope.interface.verify import verifyClass +dataDirectory = os.path.join(os.path.dirname(__file__), 'testdata') + + class Test(unittest.TestCase): "Basic tests for the loops.external package." diff --git a/setup.py b/setup.py index 33b1355..f9636cc 100644 --- a/setup.py +++ b/setup.py @@ -165,6 +165,33 @@ class SetupManager(object): else: concept.assignChild(child, predicate) + def addResource(self, name, title, resourceType, description=u'', **kw): + if name in self.resources: + self.log("Resource '%s' ('%s') already exists." % (name, title)) + c = self.resources[name] + if c.resourceType != resourceType: + self.log("Wrong resource type for '%s': '%s' instead of '%s'." % + (name, getName(c.resourceType), getName(resourceType))) + else: + c = addAndConfigureObject(self.resources, Resource, name, title=title, + description=description, + resourceType=resourceType, **kw) + self.log("Resource '%s' ('%s') created." % (name, title)) + return c + + def assignResource(self, conceptName, resourceName, predicate=None): + if predicate is None: + predicate = self.concepts.getDefaultPredicate() + if isinstance(predicate, basestring): + predicate = self.concepts[predicate] + concept = self.concepts[conceptName] + resource = self.resources[resourceName] + if resource in concept.getResources([predicate]): + self.log("Concept '%s' is already assigned to '%s with predicate '%s'.'" % + (conceptName, resourceName, getName(predicate))) + else: + concept.assignResource(resource, predicate) + def addNode(self, name, title, container=None, nodeType='page', description=u'', body=u'', targetName=None, **kw): if container is None: diff --git a/util.py b/util.py index 373d2e1..ae3fad6 100644 --- a/util.py +++ b/util.py @@ -22,12 +22,14 @@ Utility functions. $Id$ """ +import os from zope import component from zope.app.intid.interfaces import IIntIds from zope.interface import directlyProvides, directlyProvidedBy from zope.i18nmessageid import MessageFactory from zope.schema import vocabulary +import cybertools from loops.browser.util import html_quote _ = MessageFactory('loops') @@ -94,3 +96,14 @@ def getUidForObject(obj): intIds = component.getUtility(IIntIds) return str(intIds.queryId(obj)) + +def getVarDirectory(request=None): + instanceHome = None + if request is not None: + pub = request.publication + if pub is not None: + instanceHome = os.path.dirname(pub.db.getName()) + if instanceHome is None: + instanceHome = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(cybertools.__file__)))) + return os.path.join(instanceHome, 'var')