diff --git a/external/base.py b/external/base.py index 5610ba5..fd63f27 100644 --- a/external/base.py +++ b/external/base.py @@ -68,20 +68,20 @@ class BaseLoader(object): self.summary['count'] += 1 def error(self, message): - self.transcript.write(message + '\n') + self.transcript.write(message.encode('UTF-8') + '\n') self.errors.append(message) self.summary['errors'] += 1 self.logger.error(message) def warn(self, message): - self.transcript.write(message + '\n') + self.transcript.write(message.encode('UTF-8') + '\n') self.errors.append(message) self.summary['warnings'] += 1 self.logger.warn(message) def change(self, message=None): if message is not None: - self.transcript.write(message + '\n') + self.transcript.write(message.encode('UTF-8') + '\n') self.changes.append(message) self.logger.info(message) self.summary['changed'] += 1 diff --git a/external/dsv.py b/external/dsv.py index 0007ac5..6b23c55 100644 --- a/external/dsv.py +++ b/external/dsv.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2009 Helmut Merz helmutm@cy55.de +# Copyright (c) 2010 Helmut Merz helmutm@cy55.de # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -48,7 +48,7 @@ class CsvReader(BaseReader): for x in range(self.start or 0): input.readline() # skip lines on top reader = csv.DictReader(input, self.fieldNames) - lastIdentifiers = {} + allElements = {} rows = list(reader)[:self.stop] if self.sortKey: rows.sort(key=self.sortKey) @@ -72,13 +72,20 @@ class CsvReader(BaseReader): continue element = currentElements[type] = ef() element.type = type - element[k] = v # ?TODO: unmarshall + if isinstance(v, str): + v = v.decode(self.encoding) + element[k] = v for element in sorted(currentElements.values(), key=lambda x: x.order): - id = element.identifier - if not id or id != lastIdentifiers.get(element.type): - element.setParent(currentElements) + if element.identifier is None: result.append(element) - lastIdentifiers[element.type] = id + element.setParent(currentElements, allElements) + else: + typeEntry = allElements.setdefault(element.type, {}) + existing = typeEntry.get(element.identifier) + if existing is None: + typeEntry[element.identifier] = element + result.append(element) + element.setParent(currentElements, allElements) return result def ignoreRow(self, idx, row): diff --git a/external/element.py b/external/element.py index 182ffd2..7e17a0e 100644 --- a/external/element.py +++ b/external/element.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2009 Helmut Merz helmutm@cy55.de +# Copyright (c) 2010 Helmut Merz helmutm@cy55.de # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -39,6 +39,7 @@ class Element(dict): implements(IElement) + encoding = 'UTF-8' type = '' identifierName = '' object = None @@ -49,7 +50,8 @@ class Element(dict): @property def identifier(self): - return self.get(self.identifierName) + id = self.get(self.identifierName) + return id def __getitem__(self, key): if isinstance(key, Element): @@ -70,10 +72,18 @@ class Element(dict): self.subElements.append(element) element.parent = self - def setParent(self, elementsMapping): - parent = elementsMapping.get(self.parentType) - if parent is not None: - parent.add(self) + def setParent(self, elementsMapping, allElements=None): + pt = self.parentType + if pt: + pCurr = elementsMapping.get(pt) + if pCurr is not None: + if allElements and pCurr.identifier: + parent = allElements.get(pt, {}).get(pCurr.identifier) + else: + parent = pCurr + if parent is not None: + parent.add(self) + def execute(self, loader): pass