work in progress: CSV file import

git-svn-id: svn://svn.cy55.de/Zope3/src/cybertools/trunk@3344 fd906abe-77d9-0310-91a1-e0d9ade77398

external/base.py

@@ -27,15 +27,31 @@ from zope import component
 from zope.interface import implements
 from zope.cachedescriptors.property import Lazy
 
-from cybertools.external.interfaces import IImporter
+from cybertools.external.interfaces import IReader, ILoader
 
 
-class BaseImporter(object):
+class BaseReader(object):
 
-    implements(IImporter)
+    implements(IReader)
+
+    def __init__(self, context):
+        self.context = context
+
+    def read(self, input):
+        return []
+
+
+class BaseLoader(object):
+
+    implements(ILoader)
 
     transcript = u''
 
     def __init__(self, context):
         self.context = context
         self.changes = []
+        self.errors = []
+        self.summary = dict(count=0, new=0, changed=0, errors=0, warnings=0)
+
+    def load(self, elements):
+        pass

external/csv.py

@@ -1,36 +0,0 @@
-#
-#  Copyright (c) 2009 Helmut Merz helmutm@cy55.de
-#
-#  This program is free software; you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation; either version 2 of the License, or
-#  (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software
-#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#
-"""
-Base implementation for import adapters.
-
-$Id$
-"""
-
-import csv
-
-from zope import component
-from zope.interface import implements
-from zope.cachedescriptors.property import Lazy
-
-from cybertools.external.base import BaseImporter
-
-
-class CsvImporter(BaseImporter):
-
-    pass
-

external/dsv.py

@@ -0,0 +1,66 @@
+#
+#  Copyright (c) 2009 Helmut Merz helmutm@cy55.de
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+"""
+Base implementation for import adapters.
+
+$Id$
+"""
+
+import csv
+
+from zope import component
+from zope.interface import implements
+from zope.cachedescriptors.property import Lazy
+
+from cybertools.external.base import BaseReader
+from cybertools.external.element import Element
+
+
+class CsvReader(BaseReader):
+
+    elementFactories = {None: Element}
+    fieldNames = ()
+    start = stop = None
+
+    def read(self, input):
+        result = []
+        reader = csv.DictReader(input, self.fieldNames)
+        lastIdentifiers = {}
+        for idx, row in enumerate(list(reader)[self.start:self.stop]):
+            currentElements = {}
+            for k, v in row.items():
+                if k is None:
+                    continue
+                type = None
+                if '.' in k:
+                    type, k = k.split('.', 1)
+                element = currentElements.get(type)
+                if element is None:
+                    ef = self.elementFactories.get(type)
+                    if ef is None:
+                        raise ValueError('Missing element factory for %r.' % type)
+                    element = currentElements[type] = ef()
+                    element.type = type
+                element[k] = v      # ?TODO: unmarshall
+            for element in sorted(currentElements.values(), key=lambda x: x.order):
+                id = element.identifier
+                if not id or id != lastIdentifiers.get(element.type):
+                    element.setParent(currentElements)
+                    result.append(element)
+                    lastIdentifiers[element.type] = id
+        return result

external/element.py

@@ -0,0 +1,82 @@
+#
+#  Copyright (c) 2009 Helmut Merz helmutm@cy55.de
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+
+"""
+Basic implementation of the elements used for the intermediate format for export
+and import of objects.
+
+$Id$
+"""
+
+import os
+from zope import component
+from zope.cachedescriptors.property import Lazy
+from zope.dottedname.resolve import resolve
+from zope.interface import Interface, implements
+from zope.traversing.api import getName, traverse
+
+from cybertools.composer.interfaces import IInstance
+from cybertools.composer.schema.interfaces import ISchemaFactory
+from cybertools.external.interfaces import IElement
+
+
+class Element(dict):
+
+    implements(IElement)
+
+    type = ''
+    identifierName = ''
+    object = None
+    parent = None
+    subElements = None
+    parentType = ''
+    order = 0
+
+    @property
+    def identifier(self):
+        return self.get(self.identifierName)
+
+    def __getitem__(self, key):
+        if isinstance(key, Element):
+            key = (key,)
+        if isinstance(key, tuple):
+            for item in key:
+                item.parent = self
+                self.add(item)
+            return key
+        return super(Element, self).__getitem__(key)
+
+    def processExport(self, extractor):
+        pass
+
+    def add(self, element):
+        if self.subElements is None:
+            self.subElements = []
+        self.subElements.append(element)
+        element.parent = self
+
+    def setParent(self, elementsMapping):
+        parent = elementsMapping.get(self.parentType)
+        if parent is not None:
+            parent.add(self)
+
+    def execute(self, loader):
+        pass
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, dict.__repr__(self))

external/interfaces.py

@@ -24,9 +24,52 @@ $Id$
 from zope.interface import Attribute, Interface
 
 
-class IImporter(Interface):
-    """ Parses an input file or string and creates one or more corresponding
-        objects or sets the attributes of one or more existing objects.
+
+class IElement(Interface):
+    """ A dicionary-like information element that is able to represent an
+        object, a relation between objects or a special attribute.
+        The attributes of the object are represented by items of
+        the dictionary; the attribute values may be strings, unicode strings,
+        or IElement objects.
+    """
+
+    elementType = Attribute('A string denoting the element type.')
+    object = Attribute('The object that has been created from this '
+                'element during import.')
+    parent = Attribute('An optional parent element that this element is part of.')
+    subElements = Attribute('An optional list of sub-elements; initially None.')
+
+    def processExport(extractor):
+        """ Will be called by the extractor during export to allow for
+            special handling e.g. of certain attributes.
+        """
+
+    def add(element):
+        """ Add a sub-element, may be called by the extractor during export.
+        """
+
+    def execute(loader):
+        """ Create the object that is specified by the element in the
+            context of the loader and return it.
+        """
+
+
+# import functionality
+
+class IReader(Interface):
+    """ Provides objects in an intermediate format from an external source.
+        Will typically be implemented by an utility or an adapter.
+    """
+
+    def read(input):
+        """ Retrieve content from the external source returning a sequence
+            of IElement objects.
+        """
+
+
+class ILoader(Interface):
+    """ Inserts data provided by an IReader object into the
+        object space of the context object. Will typically be used as an adapter.
     """
 
     transcript = Attribute('A string describing the result of the '
@@ -41,7 +84,35 @@ class IImporter(Interface):
                     'of newly created and changed objects and the '
                     'number of errors.')
 
-    def load(file):
-        """ Load (import) data from the file given; create objects if
-            necessary.
+    def load(elements):
+        """ Create the objects and relations specified by the ``elements``
+            argument given.
         """
+
+
+# export functionality
+
+class IWriter(Interface):
+    """ Transforms object information to an external storage.
+    """
+
+    def write(elements):
+        """ Write the sequence of elements given in an external format.
+        """
+
+
+class IExtractor(Interface):
+    """ Extracts information from objects and provides them as
+        IElement objects. Will typically be used as an adapter on a
+        top-level or root object.
+    """
+
+    def extract():
+        """ Creates and returns a sequence of IElement objects by scanning
+            the content of the context object.
+        """
+
+class ISubExtractor(IExtractor):
+    """ Used for extracting special informations from individual objects
+        that will be represented by sub-elements.
+    """