work in progress: loops site synchronization: export (and subsequent import) of changes

git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@3730 fd906abe-77d9-0310-91a1-e0d9ade77398

external/base.py

@@ -39,7 +39,7 @@ from cybertools.typology.interfaces import IType
 from loops.common import adapted
 from loops.external.interfaces import ILoader, IExtractor, ISubExtractor
 from loops.external.element import elementTypes
-from loops.interfaces import IConceptSchema, IResourceSchema, IResource
+from loops.interfaces import IConceptSchema, IResourceSchema, IResource, IConcept
 from loops.layout.base import LayoutNode
 from loops.resource import Document, MediaAsset
 from loops.setup import SetupManager
@@ -113,9 +113,10 @@ class Extractor(Base):
             self.count += 1
             yield element
 
-    def extractConcepts(self):
+    def extractConcepts(self, types=None):
         for name, obj in self.concepts.items():
             if obj.conceptType != self.typeConcept:
+                if self.checkTypes(obj, types):
                     self.count += 1
                     yield self.getConceptElement(name, obj)
 
@@ -161,7 +162,8 @@ class Extractor(Base):
                 yield elem
 
     def extractChanges(self, changedSince, parents=None, predicates=None,
-                          includeSubconcepts=False, includeResources=False,):
+                       types=None,):
+                       #includeSubconcepts=False, includeResources=False,):
         changes = self.getChangeRecords()
         if not changes:
             return
@@ -173,6 +175,8 @@ class Extractor(Base):
             obj = util.getObjectForUid(tr.taskId)
             action = tr.data.get('action')
             if action in ('add', 'modify'):
+                if not self.checkTypes(obj, types):
+                    continue
                 if not self.checkParents(obj, parents, predicates):
                     continue
                 if obj not in objects:
@@ -188,6 +192,9 @@ class Extractor(Base):
                 if (not self.checkParents(obj, parents, predicates) and
                     not self.checkParents(child, parents, predicates)):
                     continue
+                if (not self.checkTypes(obj, types) and
+                    not self.checkTypes(child, types)):
+                    continue
                 if action == 'assign':
                     element = self.getAssignmentElement(obj, child, pred)
                 else:
@@ -196,7 +203,7 @@ class Extractor(Base):
                     yield element
         # TODO: include children and resources if corresponding flags are set.
 
-    def extractForParents(self, parents, predicates=None,
+    def extractForParents(self, parents, predicates=None, types=None,
                           includeSubconcepts=False, includeResources=False,):
         concepts = set(parents)
         for p in parents:
@@ -204,6 +211,7 @@ class Extractor(Base):
         conceptList = sorted(concepts, key=lambda x:
                                 (x.conceptType != self.typeConcept, getName(x)))
         for c in conceptList:
+            if self.checkTypes(c, types):
                 self.count += 1
                 yield self.getConceptElement(getName(c), c)
         for c in conceptList:
@@ -216,6 +224,7 @@ class Extractor(Base):
             for c in conceptList:
                 for obj in c.getResources(predicates):
                     if obj not in resources:
+                        if self.checkTypes(obj, types):
                             resources.add(obj)
                             self.count += 1
                             yield self.getResourceElement(getName(obj), obj)
@@ -243,12 +252,21 @@ class Extractor(Base):
     def checkParents(self, obj, parents, predicates):
         if not parents:
             return True
+        if (not IResource.providedBy(obj) and not IConcept.providedBy(obj)):
+            return False
         objParents = obj.getParents(predicates)
         for p in parents:
             if p in objParents:
                 return True
         return False
 
+    def checkTypes(self, obj, types):
+        if not types:
+            return True
+        if (not IResource.providedBy(obj) and not IConcept.providedBy(obj)):
+            return False
+        return obj.getType() in types
+
     def getConceptOrResourceElement(self, name, obj):
         if IResource.providedBy(obj):
             return self.getResourceElement(name, obj)

external/browser.py

@@ -33,6 +33,7 @@ from zope.cachedescriptors.property import Lazy
 from zope.security.proxy import removeSecurityProxy
 from zope.traversing.api import getName, getPath
 
+from cybertools.util.date import str2timeStamp
 from loops.external.base import Loader, Extractor
 from loops.external.interfaces import IReader, IWriter
 from loops import util
@@ -73,7 +74,7 @@ class ExportImport(object):
 
     def export(self):
         form = self.request.form
-        parents = predicates = None
+        parents = predicates = types = None
         parentIds = form.get('parents')
         if parentIds:
             parentIds = [id for id in parentIds.splitlines() if id]
@@ -82,6 +83,9 @@ class ExportImport(object):
         predicateIds = form.get('predicates')
         if predicateIds:
             predicates = ([self.conceptManager[id] for id in predicateIds])
+        typeIds = form.get('types')
+        if typeIds:
+            types = ([self.conceptManager[id] for id in typeIds])
         changed = form.get('changed')
         includeSubconcepts = form.get('include_subconcepts')
         includeResources = form.get('include_resources')
@@ -89,13 +93,13 @@ class ExportImport(object):
         if changed:
             changed = self.parseDate(changed)
             if changed:
-                elements = extractor.extractChanges(changed, parents, predicates,
+                elements = extractor.extractChanges(changed, parents,
-                                        includeSubconcepts, includeResources)
+                                    predicates, types)
         elif parents:
-            elements = extractor.extractForParents(parents, predicates,
+            elements = extractor.extractForParents(parents, predicates, types,
                                     includeSubconcepts, includeResources)
         else:
-            elements = extractor.extract()
+            elements = extractor.extract(types)
         return self.download(elements)
 
     def download(self, elements):
@@ -107,26 +111,25 @@ class ExportImport(object):
         self.setDownloadHeader(self.request, text)
         return text
 
-    def parseDate(self, s):
-        try:
-            t = time.strptime(s, '%Y-%m-%d %H:%M:%S')
-        except ValueError:
-            try:
-                t = time.strptime(s, '%Y-%m-%d %H:%M')
-            except ValueError:
-                t = time.strptime(s, '%Y-%m-%d')
-        return int(time.mktime(t))
-
     @Lazy
     def conceptManager(self):
         return  self.context.getConceptManager()
 
+    @Lazy
+    def typePredicate(self):
+        return self.conceptManager.getTypePredicate()
+
+    @Lazy
+    def types(self):
+        ttype = self.conceptManager['type']
+        return [dict(name=getName(p), title=p.title)
+                for p in ttype.getChildren([self.typePredicate])]
+
     @Lazy
     def predicates(self):
         ptype = self.conceptManager['predicate']
-        hasType = self.conceptManager['hasType']
         return [dict(name=getName(p), title=p.title)
-                for p in ptype.getChildren([hasType])]
+                for p in ptype.getChildren([self.typePredicate])]
 
     def upload(self):
         data = self.request.get('field.data', None)
@@ -148,3 +151,5 @@ class ExportImport(object):
         response.setHeader('Content-Type', 'text/plain')
         response.setHeader('Content-Length', len(text))
 
+    def parseDate(self, s):
+        return str2timeStamp(s)

external/exportimport.pt

@@ -59,9 +59,17 @@
               <label for="include_resources">Include Assigned Resources</label>
             </td>
           </tr>
+          <tr>
+            <td colspan="4">
+              <label for="types">Restrict Type</label><br />
+              <select multiple name="types:list" id="types"
+                      size="4">
+                <option tal:repeat="type view/types"
+                        tal:attributes="value type/name"
+                        tal:content="type/title">type</option></select>
+          </tr>
         </table>
       </div>
-      <div>&nbsp;</div>
       <div class="row">
         <div class="controls">
           <input type="submit" name="loops.export" value="Export" />

system/sync/browser.py

@@ -22,7 +22,6 @@ view class(es) for import/export.
 $Id$
 """
 
-from cStringIO import StringIO
 import os
 import time
 from zope import component
@@ -31,14 +30,18 @@ from zope.app.pagetemplate.viewpagetemplatefile import ViewPageTemplateFile
 from zope.cachedescriptors.property import Lazy
 from zope.traversing.api import getName, getPath
 
-from loops.organize.tracking.report import RecentChanges
+from cybertools.browser.form import FormController
+from cybertools.util.date import str2timeStamp, formatTimeStamp
+from loops.browser.concept import ConceptView
+from loops.external.base import Extractor
+from loops.external.interfaces import IWriter
 from loops import util
 
 
 control_macros = ViewPageTemplateFile('control.pt')
 
 
-class SyncChanges(RecentChanges):
+class SyncChanges(ConceptView):
     """ View for controlling the transfer of changes from a loops site
         to another one.
     """
@@ -47,3 +50,54 @@ class SyncChanges(RecentChanges):
     def macro(self):
         return control_macros.macros['main']
 
+    @Lazy
+    def changed(self):
+        return (self.request.get('changed_since') or
+                formatTimeStamp(self.lastSyncTimeStamp))
+
+    @Lazy
+    def lastSyncTime(self):
+        if self.lastSyncTimeStamp is None:
+            return u'-'
+        return formatTimeStamp(self.lastSyncTimeStamp)
+
+    @Lazy
+    def lastSyncTimeStamp(self):
+        return None
+
+
+class ChangesSave(FormController):
+
+    @Lazy
+    def baseDirectory(self):
+        return util.getVarDirectory(self.request)
+
+    @Lazy
+    def sitePath(self):
+        return getPath(self.view.virtualTargetObject)[1:].replace('/', '_')
+
+    @Lazy
+    def exportDirectory(self):
+        return os.path.join(self.baseDirectory, 'export', self.sitePath)
+
+    @Lazy
+    def targetView(self):
+        return self.view.virtualTarget
+
+    def update(self):
+        typeIds = self.targetView.options('types')
+        types = [self.view.conceptManager[t] for t in typeIds]
+        since = self.request.get('changed_since')
+        changed = since and str2timeStamp(since) or self.targetView.lastSyncTimeStamp
+        extractor = Extractor(self.view.loopsRoot, self.exportDirectory)
+        elements = extractor.extractChanges(changed, types)
+        writer = component.getUtility(IWriter)
+        f = open(os.path.join(self.exportDirectory, '_changes.dmp'), 'w')
+        writer.write(elements, f)
+        f.close()
+        return True
+
+
+class ChangesSync(ChangesSave):
+
+    pass

system/sync/configure.zcml

@@ -13,4 +13,16 @@
         class="loops.system.sync.browser.SyncChanges"
         permission="zope.ManageContent" />
 
+  <zope:adapter name="changes_save"
+        for="loops.browser.node.NodeView
+             zope.publisher.interfaces.browser.IBrowserRequest"
+        factory="loops.system.sync.browser.ChangesSave"
+        permission="zope.ManageContent" />
+
+  <zope:adapter name="changes_sync"
+        for="loops.browser.node.NodeView
+             zope.publisher.interfaces.browser.IBrowserRequest"
+        factory="loops.system.sync.browser.ChangesSync"
+        permission="zope.ManageContent" />
+
 </configure>

system/sync/control.pt

@@ -1,10 +1,29 @@
 <!-- $Id$ -->
 
 
-<metal:main define-macro="main"
+<metal:main define-macro="main">
-              tal:define="info item/getData">
+  <form method="post"
+        tal:define="action request/form.action|string:changes_save">
     <metal:title use-macro="item/conceptMacros/concepttitle" />
     <br />
-  <metal:listing use-macro="info/macro" />
+    <label for="changed_since" i18n:translate="">Select changes since:</label>
-  <input type="submit" value="Synchronize Changes" />
+    <input type="text" name="changed_since" id="changed_since" size="14"
+           tal:attributes="value item/changed" />&nbsp;
+    (<span i18n:translate="">Last synchronization:</span>
+    <span tal:content="item/lastSyncTime" />)
+    <br />&nbsp;
+    <div>
+      <label i18n:translate="">Action:</label>
+      <input type="radio" name="form.action" value="changes_save"
+             id="action_save"
+             tal:attributes="checked python:action == 'changes_save'" />
+      <label for="action_save" i18n:translate="">Save changes only</label>
+      <input type="radio" name="form.action" value="changes_sync"
+             id="action_sync"
+             tal:attributes="checked python:action == 'changes_sync'" />
+      <label for="action_sync" i18n:translate="">Synchronize changes</label>
+    </div>
+    <br />
+    <input type="submit" name="submit" value="Submit" i18n:translate="" />
+  </form>
 </metal:main>