avoid creating duplicate elements, fix setting of parent elements

git-svn-id: svn://svn.cy55.de/Zope3/src/cybertools/trunk@3818 fd906abe-77d9-0310-91a1-e0d9ade77398

external/base.py

@@ -68,20 +68,20 @@ class BaseLoader(object):
             self.summary['count'] += 1
 
     def error(self, message):
-        self.transcript.write(message + '\n')
+        self.transcript.write(message.encode('UTF-8') + '\n')
         self.errors.append(message)
         self.summary['errors'] += 1
         self.logger.error(message)
 
     def warn(self, message):
-        self.transcript.write(message + '\n')
+        self.transcript.write(message.encode('UTF-8') + '\n')
         self.errors.append(message)
         self.summary['warnings'] += 1
         self.logger.warn(message)
 
     def change(self, message=None):
         if message is not None:
-            self.transcript.write(message + '\n')
+            self.transcript.write(message.encode('UTF-8') + '\n')
             self.changes.append(message)
             self.logger.info(message)
         self.summary['changed'] += 1

external/dsv.py

@@ -1,5 +1,5 @@
 #
-#  Copyright (c) 2009 Helmut Merz helmutm@cy55.de
+#  Copyright (c) 2010 Helmut Merz helmutm@cy55.de
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -48,7 +48,7 @@ class CsvReader(BaseReader):
         for x in range(self.start or 0):
             input.readline()    # skip lines on top
         reader = csv.DictReader(input, self.fieldNames)
-        lastIdentifiers = {}
+        allElements = {}
         rows = list(reader)[:self.stop]
         if self.sortKey:
             rows.sort(key=self.sortKey)
@@ -72,13 +72,20 @@ class CsvReader(BaseReader):
                         continue
                     element = currentElements[type] = ef()
                     element.type = type
-                element[k] = v      # ?TODO: unmarshall
+                if isinstance(v, str):
+                    v = v.decode(self.encoding)
+                element[k] = v
             for element in sorted(currentElements.values(), key=lambda x: x.order):
-                id = element.identifier
+                if element.identifier is None:
-                if not id or id != lastIdentifiers.get(element.type):
-                    element.setParent(currentElements)
                     result.append(element)
-                    lastIdentifiers[element.type] = id
+                    element.setParent(currentElements, allElements)
+                else:
+                    typeEntry = allElements.setdefault(element.type, {})
+                    existing = typeEntry.get(element.identifier)
+                    if existing is None:
+                        typeEntry[element.identifier] = element
+                        result.append(element)
+                        element.setParent(currentElements, allElements)
         return result
 
     def ignoreRow(self, idx, row):

external/element.py

@@ -1,5 +1,5 @@
 #
-#  Copyright (c) 2009 Helmut Merz helmutm@cy55.de
+#  Copyright (c) 2010 Helmut Merz helmutm@cy55.de
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -39,6 +39,7 @@ class Element(dict):
 
     implements(IElement)
 
+    encoding = 'UTF-8'
     type = ''
     identifierName = ''
     object = None
@@ -49,7 +50,8 @@ class Element(dict):
 
     @property
     def identifier(self):
-        return self.get(self.identifierName)
+        id = self.get(self.identifierName)
+        return id
 
     def __getitem__(self, key):
         if isinstance(key, Element):
@@ -70,10 +72,18 @@ class Element(dict):
         self.subElements.append(element)
         element.parent = self
 
-    def setParent(self, elementsMapping):
+    def setParent(self, elementsMapping, allElements=None):
-        parent = elementsMapping.get(self.parentType)
+        pt = self.parentType
-        if parent is not None:
+        if pt:
-            parent.add(self)
+            pCurr = elementsMapping.get(pt)
+            if pCurr is not None:
+                if allElements and pCurr.identifier:
+                    parent = allElements.get(pt, {}).get(pCurr.identifier)
+                else:
+                    parent = pCurr
+                if parent is not None:
+                    parent.add(self)
+
 
     def execute(self, loader):
         pass