cco/cybertools
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

composer, util: Python3 fixes

Browse source
This commit is contained in:
Helmut Merz 2024-09-22 10:24:02 +02:00
parent 87ca77df45
commit 80766f2279
22 changed files with 91 additions and 958 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cybertools/composer/README.txt
View file

@ -2,8 +2,6 @@
Composer - Building Complex Structures with Templates or Schemas
================================================================
($Id$)
>>> from cybertools.composer.base import Element, Compound, Template
>>> from cybertools.composer.instance import Instance
@ -46,7 +44,7 @@ with the template.
>>> class ConfigurationAdapter(Instance):
... def applyTemplate(self):
... for c in self.template.components:
... print c, self.context.parts.get(c.name, '-')
... print(c, self.context.parts.get(c.name, '-'))
>>> inst = ConfigurationAdapter(c001)
>>> inst.template = desktop

28
cybertools/composer/instance.py
View file

@ -1,36 +1,16 @@
#
# Copyright (c) 2007 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# cybertools.composer.instance
"""
Base classes to be used for client adapters.
$Id$
""" Base classes to be used for client adapters.
"""
from zope.interface import implements
from zope.interface import implementer
from cybertools.composer.interfaces import IInstance
@implementer(IInstance)
class Instance(object):
implements(IInstance)
templateFactory = dict
templateAttributeName = '__ctc_template__'

8
cybertools/text/tests.py
View file

@ -3,13 +3,13 @@
"""
Tests for the 'cybertools.text' package.
"""
import sys
#sys.path = [p for p in sys.path if p != '']
sys.path = sys.path[2:] # avoid import cycle with bs4 when importing html
#print(sys.path)
import unittest, doctest
import sys
import warnings
#print('***', sys.path)
sys.path = sys.path[1:] # avoid import cycle with bs4 when importing html
from cybertools.text import pdf
from cybertools.text.html import htmlToText

27
cybertools/util/adapter.py
View file

@ -1,25 +1,6 @@
#
# Copyright (c) 2007 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# cybertools.util.adapter
"""
A simple adapter framework.
$Id$
""" A simple adapter framework.
"""
@ -78,7 +59,7 @@ class AdapterType(type):
factory.register(cls, adapted, name)
class AdapterBase:
class AdapterBase(object, metaclass=AdapterType):
__metaclass__ = AdapterType
pass

23
cybertools/util/cache.py
View file

@ -1,25 +1,6 @@
#
# Copyright (c) 2010 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# cybertools.util.cache
"""
A simple caching mechanism.
$Id$
""" A simple caching mechanism.
"""
from zope import component

6
cybertools/util/cache.txt
View file

@ -2,14 +2,12 @@
Data Caching
============
$Id$
>>> from cybertools.util import cache
>>> cache = cache.internalCache
>>> @cache(lambda *args: 'calc')
... def calculate():
... print 'calculating'
... print('calculating')
... return 42
>>> calculate()
@ -24,7 +22,7 @@ $Id$
... return self.id
... @cache(getId)
... def calculate(self):
... print 'calculating'
... print('calculating')
... return 42
>>> demo = Demo()

26
cybertools/util/config.py
View file

@ -1,30 +1,10 @@
#
# Copyright (c) 2007 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# cybertools.util.config
"""
Formulating configuration options.
$Id$
""" Formulating configuration options.
"""
import os
from zope.interface import implements
from cybertools.util.jeep import Jeep
@ -58,7 +38,7 @@ class Configurator(dict):
f.close()
if p is None:
return
exec p in self
exec(p, self)
def save(self, filename=None):
fn = self.getConfigFile(filename)

4
cybertools/util/config.txt
View file

@ -49,7 +49,7 @@ it with a default if not found, in one statement.
We can output a configuration in a form that is ready for loading
just by converting it to a string representation.
>>> print config
>>> print(config)
crawl[0].directory = 'documents/projects'
crawl[0].type = 'filesystem'
transport.serverURL = 'http://demo.cy55.de'
@ -69,7 +69,7 @@ for storage; normally it would be stored in the users home directory.
>>> fn
'....cybertools.cfg'
>>> print open(fn).read()
>>> print(open(fn).read())
crawl[0].directory = 'documents/projects'
crawl[0].type = 'filesystem'
transport.serverURL = 'http://demo.cy55.de'

8
cybertools/util/defer.txt
View file

@ -2,8 +2,6 @@
Deferred Execution
==================
$Id$
>>> from cybertools.util.defer import Deferred
To show what deferreds are about we need two classes.
@ -21,7 +19,7 @@ method will be called, thus notifying the client.
...
... def work(self):
... self.deferred = Deferred()
... print 'Worker: work started'
... print('Worker: work started')
... return self.deferred
...
... def nowItsTime(self):
@ -37,10 +35,10 @@ callback method with the deferred object coming back from the
... def run(self, worker):
... deferred = worker.work()
... deferred.addCallback(self.showResult)
... print 'Client: The worker seems to be working now...'
... print('Client: The worker seems to be working now...')
...
... def showResult(self, result):
... print 'Result:', result
... print('Result:', result)
So we now create a worker and a client, and let the client run:

8
cybertools/util/format.txt
View file

@ -8,12 +8,12 @@ Basic Formatting Functions
>>> time = datetime(2006, 8, 21, 17, 37, 13)
>>> format.formatDate(time, type='time', variant='medium')
u'17:37:13'
'17:37:13'
>>> format.formatDate(time, type='dateTime', variant='medium')
u'21.08.2006 17:37:13'
'21.08.2006 17:37:13'
>>> format.formatNumber(17.2)
u'17,20'
'17,20'
>>> format.formatNumber(13399.99999997)
u'13.400,00'
'13.400,00'

60
cybertools/util/html.py
View file

@ -1,29 +1,13 @@
#
# Copyright (c) 2013 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# cybertools.util.html
"""
Strip HTML tags and other HTML-related utilities.
""" Strip HTML tags and other HTML-related utilities.
"""
import re
from cybertools.text.lib.BeautifulSoup import BeautifulSoup, Comment
from cybertools.text.lib.BeautifulSoup import Declaration, NavigableString
#from cybertools.text.lib.BeautifulSoup import BeautifulSoup, Comment
#from cybertools.text.lib.BeautifulSoup import Declaration, NavigableString
from bs4 import BeautifulSoup, Comment, Declaration, NavigableString
validTags = ('a b br div em font h1 h2 h3 i img li ol p pre span strong '
'table td tr u ul').split()
@ -40,26 +24,26 @@ sentencePattern = re.compile(r'[:.\?\!]')
def sanitize(value, validTags=validTags, validAttrs=validAttrs,
validStyles=validStyles, stripEscapedComments=True):
soup = BeautifulSoup(value)
for comment in soup.findAll(text=lambda text: isinstance(text, Comment)):
soup = BeautifulSoup(value, features='lxml')
for comment in soup.findAll(string=lambda text: isinstance(text, Comment)):
comment.extract()
for tag in soup.findAll(True):
if tag.name not in validTags:
tag.hidden = True
attrs = []
for attr, val in tag.attrs:
attrs = {}
for attr, val in tag.attrs.items():
attr = attr.lower()
if attr not in validAttrs:
continue
if attr == 'style':
val = sanitizeStyle(val, validStyles)
if val:
attrs.append((attr, val))
attrs[attr] = val
tag.attrs = attrs
result = soup.renderContents()
result = soup.renderContents().decode('UTF-8')
if stripEscapedComments:
result = escCommPattern.sub(u'', result)
return result.decode('utf8')
result = escCommPattern.sub('', result)
return result
def sanitizeStyle(value, validStyles=validStyles):
@ -85,8 +69,8 @@ def checkStyle(k, validStyles=validStyles):
def stripComments(value):
soup = BeautifulSoup(value)
for comment in soup.findAll(text=lambda text: isinstance(text, Comment)):
soup = BeautifulSoup(value, features='lxml')
for comment in soup.findAll(string=lambda text: isinstance(text, Comment)):
comment.extract()
return soup.renderContents().decode('utf8')
@ -100,14 +84,14 @@ def stripAll(value):
elif tag is not None and type(tag) is not Declaration:
collectText(tag.contents)
data = []
soup = BeautifulSoup(value)
soup = BeautifulSoup(value, features='lxml')
collectText(soup.contents)
text = u''.join(data).replace(u'\n', u'').replace(u' ', u' ')
text = ''.join(data).replace('\n', '').replace(' ', ' ')
return text
def extractFirstPart(value):
soup = BeautifulSoup(value)
soup = BeautifulSoup(value, features='lxml')
for tag in soup.findAll(True):
if tag.name in ('p',):
part = tag.renderContents()
@ -115,6 +99,8 @@ def extractFirstPart(value):
else:
text = stripAll(value)
part = sentencePattern.split(text)[0]
if isinstance(part, unicode):
part = part.encode('UTF-8')
return ('<p>%s</p>' % part).decode('utf8')
#if isinstance(part, str):
# part = part.encode('UTF-8')
if isinstance(part, bytes):
part = part.decode('UTF-8')
return ('<p>%s</p>' % part) #.decode('utf8')

14
cybertools/util/html.txt
View file

@ -14,10 +14,10 @@ Sanitize HTML
-------------
>>> sanitize(input, validAttrs=['style'])
u'\n<p style="font-weight: bold">\n<a><b>Text</b>, and more</a>\n</p>\n'
'\n<p style="font-weight: bold">\n<a><b>Text</b>, and more</a>\n</p>\n'
>>> sanitize(input, ['p', 'b'], ['class'])
u'\n<p class="standard">\n<b>Text</b>, and more\n</p>\n'
'\n<p class="standard">\n<b>Text</b>, and more\n</p>\n'
All comments are stripped from the HTML input.
@ -27,18 +27,18 @@ All comments are stripped from the HTML input.
... <p>text</p>"""
>>> sanitize(input2)
u'\n<p>text</p>\n\n<p>text</p>'
'\n<p>text</p>\n\n<p>text</p>'
It's also possible to remove only the comments from the HTML input.
>>> stripComments(input2)
u'<html>\n<p>text</p>\n\n<p>text</p></html>'
'<html>\n<body><p>text</p>\n\n<p>text</p></body></html>'
It is also possible to strip all HTML tags from the input string.
>>> from cybertools.util.html import stripAll
>>> stripAll(input)
u'Text, and more'
'Text, and more'
Extract first part of an HTML text
----------------------------------
@ -46,7 +46,7 @@ Extract first part of an HTML text
>>> from cybertools.util.html import extractFirstPart
>>> extractFirstPart(input)
u'<p>\n<a href="blubb"><b>Text</b>, and more</a>\n</p>'
'<p>\n<a href="blubb"><b>Text</b>, and more</a>\n</p>'
>>> extractFirstPart(input2)
u'<p>text</p>'
'<p>text</p>'

25
cybertools/util/iterate.py
View file

@ -1,23 +1,6 @@
#
# Copyright (c) 2012 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# cybertools.util.iterate
"""
Iterator and generator utilities.
""" Iterator and generator utilities.
"""
from itertools import islice
@ -36,7 +19,7 @@ class BatchIterator(object):
def __iter__(self):
return self
def next(self):
def __next__(self):
if self.count >= (self.batch + 1) * self.limit:
raise StopIteration
if self.start:
@ -45,7 +28,7 @@ class BatchIterator(object):
self.start = 0
self.count += 1
try:
return self.data.next()
return self.data.__next__()
except StopIteration:
self.exhausted = True
raise

6
cybertools/util/iterate.txt
View file

@ -15,7 +15,7 @@ We create a BatchIterator upon a base iterator. The BatchIterator
only gives us a limited portion of the values provided by the base
iterator.
>>> it = BatchIterator(xrange(30))
>>> it = BatchIterator(range(30))
>>> list(it)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
>>> list(it)
@ -39,13 +39,13 @@ Advancing would not help if the base iterator is exhausted.
We can also immediately start at the second batch by providing the ``start``
argument to the BatchIterator constructor.
>>> it = BatchIterator(xrange(30), start=1)
>>> it = BatchIterator(range(30), start=1)
>>> list(it)
[20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
We can use another limit (i.e. the batch size) via the BatchIterator constructor.
>>> it = BatchIterator(xrange(30), start=1, limit=8)
>>> it = BatchIterator(range(30), start=1, limit=8)
>>> list(it)
[8, 9, 10, 11, 12, 13, 14, 15]
>>> it.advance()

32
cybertools/util/jeep.py
View file

@ -1,27 +1,8 @@
#
# Copyright (c) 2010 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# cybertools.util.jeep
"""
A general purpose (thus 'Jeep') class that provides most of the interfaces
""" A general purpose (thus 'Jeep') class that provides most of the interfaces
of sequences and dictionaries and in addition allows attribute access to
the dictionary entries.
$Id$
"""
_notfound = object()
@ -66,7 +47,7 @@ class Jeep(object):
super(Jeep, self).__delattr__(attr)
def __getitem__(self, key):
if type(key) in (int, long):
if isinstance(key, int):
return getattr(self, self._sequence[key])
value = getattr(self, key, _notfound)
if value is _notfound:
@ -123,7 +104,7 @@ class Jeep(object):
self[key] = value
def pop(self, key=-1, default=_undefined):
if type(key) in (int, long):
if isinstance(key, int):
key = self._sequence[key]
if default is _undefined:
value = self[key]
@ -133,7 +114,7 @@ class Jeep(object):
return value
def find(self, obj):
if isinstance(obj, basestring):
if isinstance(obj, str):
key = obj
else:
key = getattr(obj, '__name__', getattr(obj, 'name', _notfound))
@ -189,7 +170,8 @@ def moveByDelta(objs, toMove, delta):
if delta < 0:
objs = list(reversed(objs))
result.reverse()
toMove = sorted(toMove, lambda x,y: cmp(objs.index(x), objs.index(y)))
#toMove = sorted(toMove, lambda x,y: cmp(objs.index(x), objs.index(y)))
toMove = sorted(toMove, key=lambda x: objs.index(x))
for element in toMove:
newPos = min(len(result), objs.index(element) + abs(delta))
result.insert(newPos, element)

4
cybertools/util/jeep.txt
View file

@ -2,8 +2,6 @@
Jeep - a General Purpose Class
==============================
$Id$
>>> from cybertools.util.jeep import Jeep
>>> jeep = Jeep()
@ -76,7 +74,7 @@ More Dictionary Methods
KeyError: 'fourth'
>>> dict(jeep)
{'second': 'new second value', 'third': 'third value', 'first': 'first value'}
{'first': 'first value', 'second': 'new second value', 'third': 'third value'}
>>> jeep.setdefault('first', 'new first value')
'first value'

666
cybertools/util/json.py
View file

@ -1,666 +0,0 @@
"""
A simple, fast, extensible JSON encoder
JSON (JavaScript Object Notation) <http://json.org> is a subset of
JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data
interchange format."""
"""
This is a stripped-down version of simplejson
by Bob Ippolito, http://undefined.org/python/
Copyright (c) 2006 Bob Ippolito
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
import re, sys
ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
HAS_UTF8 = re.compile(r'[\x80-\xff]')
ESCAPE_DCT = {
'\\': '\\\\',
'"': '\\"',
'\b': '\\b',
'\f': '\\f',
'\n': '\\n',
'\r': '\\r',
'\t': '\\t',
}
for i in range(0x20):
ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
# Assume this produces an infinity on all machines (probably not guaranteed)
INFINITY = float('1e66666')
FLOAT_REPR = repr
def floatstr(o, allow_nan=True):
if o != o:
text = 'NaN'
elif o == INFINITY:
text = 'Infinity'
elif o == -INFINITY:
text = '-Infinity'
else:
return FLOAT_REPR(o)
if not allow_nan:
raise ValueError("Out of range float values are not JSON compliant: %r"
% (o,))
return text
def encode_basestring(s):
def replace(match):
return ESCAPE_DCT[match.group(0)]
return '"' + ESCAPE.sub(replace, s) + '"'
def py_encode_basestring_ascii(s):
if isinstance(s, str) and HAS_UTF8.search(s) is not None:
s = s.decode('utf-8')
def replace(match):
s = match.group(0)
try:
return ESCAPE_DCT[s]
except KeyError:
n = ord(s)
if n < 0x10000:
return '\\u%04x' % (n,)
else:
# surrogate pair
n -= 0x10000
s1 = 0xd800 | ((n >> 10) & 0x3ff)
s2 = 0xdc00 | (n & 0x3ff)
return '\\u%04x\\u%04x' % (s1, s2)
return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
encode_basestring_ascii = py_encode_basestring_ascii
class JSONEncoder(object):
item_separator = ', '
key_separator = ': '
def __init__(self, skipkeys=False, ensure_ascii=True,
check_circular=True, allow_nan=True, sort_keys=False,
indent=None, separators=None, encoding='utf-8', default=None):
self.skipkeys = skipkeys
self.ensure_ascii = ensure_ascii
self.check_circular = check_circular
self.allow_nan = allow_nan
self.sort_keys = sort_keys
self.indent = indent
self.current_indent_level = 0
if separators is not None:
self.item_separator, self.key_separator = separators
if default is not None:
self.default = default
self.encoding = encoding
def _newline_indent(self):
return '\n' + (' ' * (self.indent * self.current_indent_level))
def _iterencode_list(self, lst, markers=None):
if not lst:
yield '[]'
return
if markers is not None:
markerid = id(lst)
if markerid in markers:
raise ValueError("Circular reference detected")
markers[markerid] = lst
yield '['
if self.indent is not None:
self.current_indent_level += 1
newline_indent = self._newline_indent()
separator = self.item_separator + newline_indent
yield newline_indent
else:
newline_indent = None
separator = self.item_separator
first = True
for value in lst:
if first:
first = False
else:
yield separator
for chunk in self._iterencode(value, markers):
yield chunk
if newline_indent is not None:
self.current_indent_level -= 1
yield self._newline_indent()
yield ']'
if markers is not None:
del markers[markerid]
def _iterencode_dict(self, dct, markers=None):
if not dct:
yield '{}'
return
if markers is not None:
markerid = id(dct)
if markerid in markers:
raise ValueError("Circular reference detected")
markers[markerid] = dct
yield '{'
key_separator = self.key_separator
if self.indent is not None:
self.current_indent_level += 1
newline_indent = self._newline_indent()
item_separator = self.item_separator + newline_indent
yield newline_indent
else:
newline_indent = None
item_separator = self.item_separator
first = True
if self.ensure_ascii:
encoder = encode_basestring_ascii
else:
encoder = encode_basestring
allow_nan = self.allow_nan
if self.sort_keys:
keys = dct.keys()
keys.sort()
items = [(k, dct[k]) for k in keys]
else:
items = dct.iteritems()
_encoding = self.encoding
_do_decode = (_encoding is not None
and not (_encoding == 'utf-8'))
for key, value in items:
if isinstance(key, str):
if _do_decode:
key = key.decode(_encoding)
elif isinstance(key, basestring):
pass
# JavaScript is weakly typed for these, so it makes sense to
# also allow them. Many encoders seem to do something like this.
elif isinstance(key, float):
key = floatstr(key, allow_nan)
elif isinstance(key, (int, long)):
key = str(key)
elif key is True:
key = 'true'
elif key is False:
key = 'false'
elif key is None:
key = 'null'
elif self.skipkeys:
continue
else:
raise TypeError("key %r is not a string" % (key,))
if first:
first = False
else:
yield item_separator
yield encoder(key)
yield key_separator
for chunk in self._iterencode(value, markers):
yield chunk
if newline_indent is not None:
self.current_indent_level -= 1
yield self._newline_indent()
yield '}'
if markers is not None:
del markers[markerid]
def _iterencode(self, o, markers=None):
if isinstance(o, basestring):
if self.ensure_ascii:
encoder = encode_basestring_ascii
else:
encoder = encode_basestring
_encoding = self.encoding
if (_encoding is not None and isinstance(o, str)
and not (_encoding == 'utf-8')):
o = o.decode(_encoding)
yield encoder(o)
elif o is None:
yield 'null'
elif o is True:
yield 'true'
elif o is False:
yield 'false'
elif isinstance(o, (int, long)):
yield str(o)
elif isinstance(o, float):
yield floatstr(o, self.allow_nan)
elif isinstance(o, (list, tuple)):
for chunk in self._iterencode_list(o, markers):
yield chunk
elif isinstance(o, dict):
for chunk in self._iterencode_dict(o, markers):
yield chunk
else:
if markers is not None:
markerid = id(o)
if markerid in markers:
raise ValueError("Circular reference detected")
markers[markerid] = o
for chunk in self._iterencode_default(o, markers):
yield chunk
if markers is not None:
del markers[markerid]
def _iterencode_default(self, o, markers=None):
newobj = self.default(o)
return self._iterencode(newobj, markers)
def default(self, o):
raise TypeError("%r is not JSON serializable" % (o,))
def encode(self, o):
if isinstance(o, basestring):
if isinstance(o, str):
_encoding = self.encoding
if (_encoding is not None
and not (_encoding == 'utf-8')):
o = o.decode(_encoding)
if self.ensure_ascii:
return encode_basestring_ascii(o)
else:
return encode_basestring(o)
chunks = list(self.iterencode(o))
return ''.join(chunks)
def iterencode(self, o):
if self.check_circular:
markers = {}
else:
markers = None
return self._iterencode(o, markers)
_default_encoder = JSONEncoder(
skipkeys=False,
ensure_ascii=True,
check_circular=True,
allow_nan=True,
indent=None,
separators=None,
encoding='utf-8',
default=None,
)
"""
Implementation of JSONDecoder
"""
import sre_parse
import sre_compile
import sre_constants
from sre_constants import BRANCH, SUBPATTERN
FLAGS = (re.VERBOSE | re.MULTILINE | re.DOTALL)
class Scanner(object):
def __init__(self, lexicon, flags=FLAGS):
self.actions = [None]
# Combine phrases into a compound pattern
s = sre_parse.Pattern()
s.flags = flags
p = []
for idx, token in enumerate(lexicon):
phrase = token.pattern
try:
subpattern = sre_parse.SubPattern(s,
[(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))])
except sre_constants.error:
raise
p.append(subpattern)
self.actions.append(token)
s.groups = len(p) + 1 # NOTE(guido): Added to make SRE validation work
p = sre_parse.SubPattern(s, [(BRANCH, (None, p))])
self.scanner = sre_compile.compile(p)
def iterscan(self, string, idx=0, context=None):
"""
Yield match, end_idx for each match
"""
match = self.scanner.scanner(string, idx).match
actions = self.actions
lastend = idx
end = len(string)
while True:
m = match()
if m is None:
break
matchbegin, matchend = m.span()
if lastend == matchend:
break
action = actions[m.lastindex]
if action is not None:
rval, next_pos = action(m, context)
if next_pos is not None and next_pos != matchend:
# "fast forward" the scanner
matchend = next_pos
match = self.scanner.scanner(string, matchend).match
yield rval, matchend
lastend = matchend
def pattern(pattern, flags=FLAGS):
def decorator(fn):
fn.pattern = pattern
fn.regex = re.compile(pattern, flags)
return fn
return decorator
def _floatconstants():
import struct
import sys
_BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
if sys.byteorder != 'big':
_BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
nan, inf = struct.unpack('dd', _BYTES)
return nan, inf, -inf
NaN, PosInf, NegInf = _floatconstants()
def linecol(doc, pos):
lineno = doc.count('\n', 0, pos) + 1
if lineno == 1:
colno = pos
else:
colno = pos - doc.rindex('\n', 0, pos)
return lineno, colno
def errmsg(msg, doc, pos, end=None):
lineno, colno = linecol(doc, pos)
if end is None:
return '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos)
endlineno, endcolno = linecol(doc, end)
return '%s: line %d column %d - line %d column %d (char %d - %d)' % (
msg, lineno, colno, endlineno, endcolno, pos, end)
_CONSTANTS = {
'-Infinity': NegInf,
'Infinity': PosInf,
'NaN': NaN,
'true': True,
'false': False,
'null': None,
}
def JSONConstant(match, context, c=_CONSTANTS):
s = match.group(0)
fn = getattr(context, 'parse_constant', None)
if fn is None:
rval = c[s]
else:
rval = fn(s)
return rval, None
pattern('(-?Infinity|NaN|true|false|null)')(JSONConstant)
def JSONNumber(match, context):
match = JSONNumber.regex.match(match.string, *match.span())
integer, frac, exp = match.groups()
if frac or exp:
fn = getattr(context, 'parse_float', None) or float
res = fn(integer + (frac or '') + (exp or ''))
else:
fn = getattr(context, 'parse_int', None) or int
res = fn(integer)
return res, None
pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber)
STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
BACKSLASH = {
'"': u'"', '\\': u'\\', '/': u'/',
'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
}
DEFAULT_ENCODING = "utf-8"
def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match):
if encoding is None:
encoding = DEFAULT_ENCODING
chunks = []
_append = chunks.append
begin = end - 1
while 1:
chunk = _m(s, end)
if chunk is None:
raise ValueError(
errmsg("Unterminated string starting at", s, begin))
end = chunk.end()
content, terminator = chunk.groups()
if content:
if not isinstance(content, unicode):
content = unicode(content, encoding)
_append(content)
if terminator == '"':
break
elif terminator != '\\':
if strict:
raise ValueError(errmsg("Invalid control character %r at", s, end))
else:
_append(terminator)
continue
try:
esc = s[end]
except IndexError:
raise ValueError(
errmsg("Unterminated string starting at", s, begin))
if esc != 'u':
try:
m = _b[esc]
except KeyError:
raise ValueError(
errmsg("Invalid \\escape: %r" % (esc,), s, end))
end += 1
else:
esc = s[end + 1:end + 5]
next_end = end + 5
msg = "Invalid \\uXXXX escape"
try:
if len(esc) != 4:
raise ValueError
uni = int(esc, 16)
if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
if not s[end + 5:end + 7] == '\\u':
raise ValueError
esc2 = s[end + 7:end + 11]
if len(esc2) != 4:
raise ValueError
uni2 = int(esc2, 16)
uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
next_end += 6
m = unichr(uni)
except ValueError:
raise ValueError(errmsg(msg, s, end))
end = next_end
_append(m)
return u''.join(chunks), end
scanstring = py_scanstring
def JSONString(match, context):
encoding = getattr(context, 'encoding', None)
strict = getattr(context, 'strict', True)
return scanstring(match.string, match.end(), encoding, strict)
pattern(r'"')(JSONString)
WHITESPACE = re.compile(r'\s*', FLAGS)
def JSONObject(match, context, _w=WHITESPACE.match):
pairs = {}
s = match.string
end = _w(s, match.end()).end()
nextchar = s[end:end + 1]
# Trivial empty object
if nextchar == '}':
return pairs, end + 1
if nextchar != '"':
raise ValueError(errmsg("Expecting property name", s, end))
end += 1
encoding = getattr(context, 'encoding', None)
strict = getattr(context, 'strict', True)
iterscan = JSONScanner.iterscan
while True:
key, end = scanstring(s, end, encoding, strict)
end = _w(s, end).end()
if s[end:end + 1] != ':':
raise ValueError(errmsg("Expecting : delimiter", s, end))
end = _w(s, end + 1).end()
try:
value, end = iterscan(s, idx=end, context=context).next()
except StopIteration:
raise ValueError(errmsg("Expecting object", s, end))
pairs[key] = value
end = _w(s, end).end()
nextchar = s[end:end + 1]
end += 1
if nextchar == '}':
break
if nextchar != ',':
raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
end = _w(s, end).end()
nextchar = s[end:end + 1]
end += 1
if nextchar != '"':
raise ValueError(errmsg("Expecting property name", s, end - 1))
object_hook = getattr(context, 'object_hook', None)
if object_hook is not None:
pairs = object_hook(pairs)
return pairs, end
pattern(r'{')(JSONObject)
def JSONArray(match, context, _w=WHITESPACE.match):
values = []
s = match.string
end = _w(s, match.end()).end()
# Look-ahead for trivial empty array
nextchar = s[end:end + 1]
if nextchar == ']':
return values, end + 1
iterscan = JSONScanner.iterscan
while True:
try:
value, end = iterscan(s, idx=end, context=context).next()
except StopIteration:
raise ValueError(errmsg("Expecting object", s, end))
values.append(value)
end = _w(s, end).end()
nextchar = s[end:end + 1]
end += 1
if nextchar == ']':
break
if nextchar != ',':
raise ValueError(errmsg("Expecting , delimiter", s, end))
end = _w(s, end).end()
return values, end
pattern(r'\[')(JSONArray)
ANYTHING = [
JSONObject,
JSONArray,
JSONString,
JSONConstant,
JSONNumber,
]
JSONScanner = Scanner(ANYTHING)
class JSONDecoder(object):
_scanner = Scanner(ANYTHING)
__all__ = ['__init__', 'decode', 'raw_decode']
def __init__(self, encoding=None, object_hook=None, parse_float=None,
parse_int=None, parse_constant=None, strict=True):
self.encoding = encoding
self.object_hook = object_hook
self.parse_float = parse_float
self.parse_int = parse_int
self.parse_constant = parse_constant
self.strict = strict
def decode(self, s, _w=WHITESPACE.match):
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
end = _w(s, end).end()
if end != len(s):
raise ValueError(errmsg("Extra data", s, end, len(s)))
return obj
def raw_decode(self, s, **kw):
kw.setdefault('context', self)
try:
obj, end = self._scanner.iterscan(s, **kw).next()
except StopIteration:
raise ValueError("No JSON object could be decoded")
return obj, end
_default_decoder = JSONDecoder(encoding=None, object_hook=None)
# public functions
def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
allow_nan=True, cls=None, indent=None, separators=None,
encoding='utf-8', default=None, **kw):
if (skipkeys is False and ensure_ascii is True and
check_circular is True and allow_nan is True and
cls is None and indent is None and separators is None and
encoding == 'utf-8' and default is None and not kw):
return _default_encoder.encode(obj)
if cls is None:
cls = JSONEncoder
return cls(
skipkeys=skipkeys, ensure_ascii=ensure_ascii,
check_circular=check_circular, allow_nan=allow_nan, indent=indent,
separators=separators, encoding=encoding, default=default,
**kw).encode(obj)
def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None,
parse_int=None, parse_constant=None, **kw):
if (cls is None and encoding is None and object_hook is None and
parse_int is None and parse_float is None and
parse_constant is None and not kw):
return _default_decoder.decode(s)
if cls is None:
cls = JSONDecoder
if object_hook is not None:
kw['object_hook'] = object_hook
if parse_float is not None:
kw['parse_float'] = parse_float
if parse_int is not None:
kw['parse_int'] = parse_int
if parse_constant is not None:
kw['parse_constant'] = parse_constant
return cls(encoding=encoding, **kw).decode(s)

64
cybertools/util/json.txt
View file

@ -1,64 +0,0 @@
==========================
JSON Encoding and Decoding
==========================
$Id$
This is a stripped-down version of simplejson
by Bob Ippolito, http://undefined.org/python/
Copyright (c) 2006 Bob Ippolito
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
>>> from cybertools.util.json import dumps, loads
Encoding basic Python object hierarchies::
>>> dumps(['foo', {'bar': ('baz', None, 1.0, 2)}])
'["foo", {"bar": ["baz", null, 1.0, 2]}]'
>>> print dumps("\"foo\bar")
"\"foo\bar"
>>> print dumps(u'\u1234')
"\u1234"
>>> print dumps('\\')
"\\"
>>> print dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)
{"a": 0, "b": 0, "c": 0}
Compact encoding::
>>> dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':'))
'[1,2,3,{"4":5,"6":7}]'
Pretty printing::
>>> print dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)
{
"4": 5,
"6": 7
}
Decoding JSON::
>>> loads('["foo", {"bar":["baz", null, 1.0, 2]}]')
[u'foo', {u'bar': [u'baz', None, 1.0, 2]}]
>>> loads('"\\"foo\\bar"')
u'"foo\x08ar'

10
cybertools/util/multikey.txt
View file

@ -65,9 +65,9 @@ Index entries that are present in the stored dictionary must always match:
If you don't know any more what entries we had added to the registry just
query its items() method:
>>> sorted(registry.items())
[(('edit.html', None, None, 'Custom'), 'edit.html for Custom skin'),
(('edit.html', 'topic', 'zope3', 'Custom'), 'very special edit.html'),
(('index.html', None, None, None), 'global index.html'),
>>> registry.items()
((('index.html', None, None, None), 'global index.html'),
(('index.html', None, None, 'Custom'), 'Global index.html for Custom skin'),
(('index.html', 'topic', None, None), 'index.html for type "topic"')]
(('index.html', 'topic', None, None), 'index.html for type "topic"'),
(('edit.html', 'topic', 'zope3', 'Custom'), 'very special edit.html'),
(('edit.html', None, None, 'Custom'), 'edit.html for Custom skin'))

4
cybertools/util/property.txt
View file

@ -2,8 +2,6 @@
Smart Properties
================
$Id$
lzprop
======
@ -21,7 +19,7 @@ by printing an informative message:
... base = 6
... @lzprop
... def value(self):
... print 'calculating'
... print('calculating')
... return self.base * 7
>>> demo = Demo()

18
cybertools/util/tests.py
View file

@ -1,25 +1,28 @@
# $Id$
# cybertools.util.tests
import unittest
import doctest
import cybertools.util.property
import unittest, doctest
import sys
import warnings
#print('***', sys.path)
sys.path = sys.path[1:]
class Test(unittest.TestCase):
"Basic tests for modules in the util package."
def testBasicStuff(self):
warnings.filterwarnings('ignore', category=ResourceWarning)
warnings.filterwarnings('ignore', category=DeprecationWarning)
pass
def test_suite():
flags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
return unittest.TestSuite((
#unittest.makeSuite(Test), # we don't need this
unittest.makeSuite(Test),
#doctest.DocTestSuite(cybertools.util.property, optionflags=flags),
doctest.DocFileSuite('adapter.txt', optionflags=flags),
doctest.DocFileSuite('aop.txt', optionflags=flags),
#doctest.DocFileSuite('aop.txt', optionflags=flags),
doctest.DocFileSuite('cache.txt', optionflags=flags),
doctest.DocFileSuite('config.txt', optionflags=flags),
doctest.DocFileSuite('defer.txt', optionflags=flags),
@ -28,7 +31,6 @@ def test_suite():
doctest.DocFileSuite('iterate.txt', optionflags=flags),
doctest.DocFileSuite('multikey.txt', optionflags=flags),
doctest.DocFileSuite('property.txt', optionflags=flags),
doctest.DocFileSuite('json.txt', optionflags=flags),
doctest.DocFileSuite('jeep.txt', optionflags=flags),
doctest.DocFileSuite('randomname.txt', optionflags=flags),
doctest.DocFileSuite('version.txt', optionflags=flags),

4
cybertools/util/version.txt
View file

@ -2,8 +2,6 @@
Collect Version Information from Different Packages
===================================================
$Id$
>>> from cybertools.util.version import versions
>>> versions.add()
@ -11,7 +9,7 @@ $Id$
>>> v = versions.get('cybertools.util.version')
>>> v
cybertools.util.version 0.4-3014
>>> print v
>>> print(v)
0.4-3014
>>> v.short
'0.4'