231 lines
8.8 KiB
Python
231 lines
8.8 KiB
Python
#
|
|
# Copyright (c) 2009 Helmut Merz helmutm@cy55.de
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
#
|
|
|
|
"""
|
|
Concept adapter(s) for external collections, e.g. a directory in the
|
|
file system.
|
|
|
|
$Id$
|
|
"""
|
|
|
|
from datetime import datetime
|
|
from logging import getLogger
|
|
import os, re, stat
|
|
|
|
from zope.app.container.interfaces import INameChooser
|
|
from zope.cachedescriptors.property import Lazy
|
|
from zope import component
|
|
from zope.component import adapts
|
|
from zope.contenttype import guess_content_type
|
|
from zope.event import notify
|
|
from zope.lifecycleevent import ObjectModifiedEvent
|
|
from zope.interface import implements, Attribute
|
|
from zope.schema.interfaces import IField
|
|
from zope.traversing.api import getName, getParent
|
|
|
|
from cybertools.meta.interfaces import IOptions
|
|
from cybertools.text import mimetypes
|
|
from cybertools.typology.interfaces import IType
|
|
from loops.common import AdapterBase, adapted, normalizeName
|
|
from loops.interfaces import IResource, IConcept
|
|
from loops.integrator.interfaces import IExternalCollection
|
|
from loops.integrator.interfaces import IExternalCollectionProvider
|
|
from loops.resource import Resource
|
|
from loops.setup import addAndConfigureObject
|
|
from loops.type import TypeInterfaceSourceList
|
|
from loops.versioning.interfaces import IVersionable
|
|
|
|
|
|
TypeInterfaceSourceList.typeInterfaces += (IExternalCollection,)
|
|
|
|
|
|
class ExternalCollectionAdapter(AdapterBase):
|
|
""" A concept adapter for accessing an external collection.
|
|
May delegate access to a named utility.
|
|
"""
|
|
|
|
implements(IExternalCollection)
|
|
adapts(IConcept)
|
|
|
|
_adapterAttributes = ('context', '__parent__', 'exclude', 'newResources')
|
|
_contextAttributes = list(IExternalCollection) + list(IConcept)
|
|
|
|
newResources = None
|
|
|
|
def getExclude(self):
|
|
return getattr(self.context, '_exclude', None) or []
|
|
def setExclude(self, value):
|
|
self.context._exclude = value
|
|
exclude = property(getExclude, setExclude)
|
|
|
|
def update(self):
|
|
existing = self.context.getResources()
|
|
old = dict((adapted(obj).externalAddress, obj) for obj in existing)
|
|
versions = set()
|
|
if self.useVersioning:
|
|
for obj in old.values():
|
|
for vaddr, vobj, vid in self.getVersions(obj):
|
|
print '###', vaddr, vobj, vid
|
|
versions.add(vaddr)
|
|
new = []
|
|
oldFound = []
|
|
provider = component.getUtility(IExternalCollectionProvider,
|
|
name=self.providerName or '')
|
|
#print '*** old', old, versions, self.lastUpdated
|
|
for addr, mdate in provider.collect(self):
|
|
#print '***', addr, mdate
|
|
if addr in versions:
|
|
continue
|
|
if addr in old:
|
|
# may be it would be better to return a file's hash
|
|
# for checking for changes...
|
|
oldFound.append(addr)
|
|
if self.lastUpdated is None or (mdate and mdate > self.lastUpdated):
|
|
obj = old[addr]
|
|
# update settings and regenerate scale variant for media asset
|
|
adobj = adapted(obj)
|
|
directory = provider.getDirectory(self)
|
|
adobj.storageParams=dict(subdirectory=directory)
|
|
adobj.externalAddress = addr
|
|
# force reindexing
|
|
notify(ObjectModifiedEvent(obj))
|
|
else:
|
|
new.append(addr)
|
|
#print '*** new', new
|
|
if new:
|
|
self.newResources = provider.createExtFileObjects(self, new)
|
|
for r in self.newResources:
|
|
self.context.assignResource(r)
|
|
for addr in old:
|
|
if str(addr) not in oldFound:
|
|
# not part of the collection any more
|
|
# TODO: only remove from collection but keep object?
|
|
self.remove(old[addr])
|
|
for r in self.context.getResources():
|
|
adobj = adapted(r)
|
|
if self.metaInfo != adobj.metaInfo and (
|
|
not adobj.metaInfo or self.overwriteMetaInfo):
|
|
adobj.metaInfo = self.metaInfo
|
|
self.lastUpdated = datetime.today()
|
|
|
|
def clear(self):
|
|
for obj in self.context.getResources():
|
|
self.remove(obj)
|
|
|
|
def remove(self, obj):
|
|
del self.resourceManager[getName(obj)]
|
|
|
|
@Lazy
|
|
def resourceManager(self):
|
|
return self.context.getLoopsRoot().getResourceManager()
|
|
|
|
@Lazy
|
|
def useVersioning(self):
|
|
if IOptions(self.context.getLoopsRoot())('useVersioning'):
|
|
return True
|
|
|
|
def getVersions(self, obj):
|
|
versionable = IVersionable(obj)
|
|
return [(adapted(v).externalAddress, v, vid)
|
|
#for vid, v in versionable.versions.items() if vid != '1.1']
|
|
for vid, v in versionable.versions.items()
|
|
if IVersionable(v).parent is not None]
|
|
|
|
|
|
class DirectoryCollectionProvider(object):
|
|
""" A utility that provides access to files in a directory.
|
|
"""
|
|
|
|
implements(IExternalCollectionProvider)
|
|
|
|
extFileTypeMapping = {
|
|
'image/*': 'media_asset',
|
|
'*/*': 'extfile',
|
|
}
|
|
|
|
def collect(self, client):
|
|
directory = self.getDirectory(client)
|
|
pattern = re.compile(client.pattern or '.*')
|
|
for path, dirs, files in os.walk(directory):
|
|
if client.excludeDirectories:
|
|
del dirs[:]
|
|
if '.svn' in dirs:
|
|
del dirs[dirs.index('.svn')]
|
|
for ex in client.exclude:
|
|
if ex in dirs:
|
|
del dirs[dirs.index(ex)]
|
|
for f in files:
|
|
if pattern.match(f):
|
|
mtime = os.stat(os.path.join(path, f))[stat.ST_MTIME]
|
|
yield (os.path.join(path[len(directory)+1:], f),
|
|
datetime.fromtimestamp(mtime))
|
|
|
|
def createExtFileObjects(self, client, addresses, extFileTypes=None):
|
|
if extFileTypes is None:
|
|
cm = client.context.getLoopsRoot().getConceptManager()
|
|
extFileTypes = dict((k, cm.get(v))
|
|
for k, v in self.extFileTypeMapping.items())
|
|
container = client.context.getLoopsRoot().getResourceManager()
|
|
directory = self.getDirectory(client)
|
|
for addr in addresses:
|
|
name = self.generateName(container, addr)
|
|
title = self.generateTitle(addr)
|
|
contentType = guess_content_type(addr,
|
|
default='application/octet-stream')[0]
|
|
extFileType = extFileTypes.get(contentType)
|
|
if extFileType is None:
|
|
extFileType = extFileTypes.get(contentType.split('/')[0] + '/*')
|
|
if extFileType is None:
|
|
extFileType = extFileTypes['*/*']
|
|
if extFileType is None:
|
|
getLogger('loops.integrator.collection.DirectoryCollectionProvider'
|
|
).warn('No external file type found for %r, '
|
|
'content type: %r' % (name, contentType))
|
|
obj = addAndConfigureObject(
|
|
container, Resource, name,
|
|
title=title,
|
|
resourceType=extFileType,
|
|
storageName='fullpath',
|
|
storageParams=dict(subdirectory=directory),
|
|
contentType=contentType,
|
|
)
|
|
adobj = adapted(obj)
|
|
adobj.externalAddress = addr # must be set last
|
|
yield obj
|
|
|
|
def getDirectory(self, client):
|
|
baseAddress = client.baseAddress or ''
|
|
address = client.address or ''
|
|
return str(os.path.join(baseAddress, address))
|
|
|
|
def generateName(self, container, name):
|
|
name = INameChooser(container).chooseName(name, None)
|
|
return name
|
|
|
|
def generateTitle(self, title):
|
|
title = os.path.split(title)[-1]
|
|
if '.' in title:
|
|
base, ext = title.rsplit('.', 1)
|
|
if ext.lower() in mimetypes.extensions.values():
|
|
title = base
|
|
if not isinstance(title, unicode):
|
|
try:
|
|
title = title.decode('UTF-8')
|
|
except UnicodeDecodeError:
|
|
title = title.decode('CP852')
|
|
return title
|