diff --git a/cco/member/README.txt b/cco/member/README.txt index 4054509..bd2f8e3 100644 --- a/cco/member/README.txt +++ b/cco/member/README.txt @@ -45,7 +45,7 @@ TAN entry form) is executed. >>> req.setTraversalStack(['++auth++2factor']) >>> scp.extractCredentials(req) - '2fa_tan_form.html?a=...&h=...&b=...' + '2fa_tan_form.html?h=...&a=...&b=...' What if we enter data for authentication phase 2? No authentication because the hashes don't match. diff --git a/cco/processor/__init__.py b/cco/processor/__init__.py new file mode 100644 index 0000000..4107094 --- /dev/null +++ b/cco/processor/__init__.py @@ -0,0 +1,2 @@ +""" package cco.processor +""" diff --git a/cco/processor/common.py b/cco/processor/common.py new file mode 100644 index 0000000..48e391d --- /dev/null +++ b/cco/processor/common.py @@ -0,0 +1,28 @@ +# +# cco.processor.common +# + +""" +Common stuff for the cco.processor package +""" + + +class Error(object): + + def __init__(self, msg): + self.message = msg + + def __str__(self): + return '' % self.message + + __repr__ = __str__ + +def error(msg): + return Error(msg) + +# TODO: create a class (with __repr__() method) for _not_found and _invalid + +_not_found = object() +_invalid = object() + + diff --git a/cco/processor/controller.py b/cco/processor/controller.py new file mode 100644 index 0000000..b56a0f3 --- /dev/null +++ b/cco/processor/controller.py @@ -0,0 +1,33 @@ +# +# cco.processor.controller +# + +from logging import getLogger + +logger = getLogger('cco.processor.controller') + + +def loop(jobname, fct, data, skip=0, limit=None, + bsize=10, action=None): + logger.info('loop %s starting' % jobname) + result = dict(count=0, created=0, updated=0, error=0) + if skip > 0: + start = skip - 1 + for i, row in enumerate(data): + if i >= start: + break + for row in data: + r = fct(row) + result['count'] += 1 + if r is not None: + result[r] += 1 + if result['count'] % bsize == 0: + logger.info('loop %s: %s' % (jobname, result)) + if action is not None: + action() + if limit and result['count'] >= limit: + break + if action is not None: + action() + logger.info('loop %s finished: %s' % (jobname, result)) + return result diff --git a/cco/processor/hook.py b/cco/processor/hook.py new file mode 100644 index 0000000..2e89374 --- /dev/null +++ b/cco/processor/hook.py @@ -0,0 +1,80 @@ +# +# cco.processor.hook +# + +""" +Mixin classes and other stuff for hooking into loops (Zope3, Bluebream) +objects so that certain operations (like setting and retrieving attributes) +can be handled by services like storage or notifiers. +""" + +from logging import getLogger +import traceback +import transaction + +from cco.processor.common import _not_found + +from loops import common + +logger = getLogger('cco.processor.hook') + +loader_hooks = {} +processor_hooks = {} + + +def loadData(obj): + #print 'loadData ***', obj.context.__name__ + data = {} + for hook in obj._hook_loaders: + fct = loader_hooks.get(hook) + try: + fct(obj, data) + except: + logger.error(traceback.format_exc()) + return data + + +def processData(obj, data): + #print 'processData ***', obj.context.__name__ + for hook in obj._hook_processors: + fct = processor_hooks.get(hook) + try: + fct(obj, data) + except: + logger.error(traceback.format_exc()) + + +class AdapterBase(common.AdapterBase): + + _hook_message_base = 'cco/data/dummy' + _hook_loaders = [] + _hook_processors = [] + _hook_config = {} + + _old_data = None + _cont = None + _id = None + + def __init__(self, context): + super(AdapterBase, self).__init__(context) + object.__setattr__(self, '_new_data', {}) + + def __getattr__(self, attr): + value = self._new_data.get(attr, _not_found) + if value is _not_found: + if self._old_data is None: + object.__setattr__(self, '_old_data', loadData(self)) + value = self._old_data.get(attr, _not_found) + if value is _not_found: + value = super(AdapterBase, self).__getattr__(attr) + self._old_data[attr] = value + return value + + def __setattr__(self, attr, value): + super(AdapterBase, self).__setattr__(attr, value) + if attr.startswith('__') or attr in self._adapterAttributes: + return + if not self._new_data: + tr = transaction.manager.get() + tr.addBeforeCommitHook(processData, [self, self._new_data], {}) + self._new_data[attr] = value diff --git a/cco/processor/importer.py b/cco/processor/importer.py new file mode 100644 index 0000000..86fe3d3 --- /dev/null +++ b/cco/processor/importer.py @@ -0,0 +1,15 @@ +# +# cco.processor.importer +# + +import csv +from os.path import join + + +def import_csv(import_dir, fn): + path = join(import_dir, fn) + with open(path) as f: + reader = csv.DictReader(f) + for row in reader: + yield row + diff --git a/cco/processor/storage.py b/cco/processor/storage.py new file mode 100644 index 0000000..eea0be4 --- /dev/null +++ b/cco/processor/storage.py @@ -0,0 +1,109 @@ +# +# cco.processor.storage +# + +from logging import getLogger +from zope.event import notify +from zope.lifecycleevent import ObjectModifiedEvent +from zope.traversing.api import getName + +from loops.common import adapted, baseObject +from loops.concept import Concept +from loops.setup import addAndConfigureObject +from loops import util + +from cco.processor.common import Error, _invalid + +logger = getLogger('cco.processor.storage') + + +# check attributes, collect changes + +def check_change(obj, attr, newValue, includeOnly=None, omit=[], updateEmpty=[]): + if attr.startswith('_') or attr in omit or newValue in (_invalid, None): + return None + if includeOnly is not None and attr not in includeOnly: + return None + oldValue = None + if obj is not None: # called from create_object + oldValue = getattr(obj, attr) + if isinstance(newValue, list) and oldValue: + oldValue = list(oldValue) + if newValue == oldValue or (oldValue and attr in updateEmpty): + return None + return (attr, (oldValue, newValue)) + +def collect_changes(obj, data, includeOnly=None, omit=[], updateEmpty=[]): + changes = [check_change(obj, attr, newValue, includeOnly, omit, updateEmpty) + for attr, newValue in data.items()] + return dict(c for c in changes if c is not None) + +# access to persistent objects + +def create_or_update_object(context, type_name, data, + includeOnly=None, omit=[], updateEmpty=[], dryRun=False): + obj = get_object(context, type_name, data) + if obj is None: + return create_object(context, type_name, data, includeOnly, omit, updateEmpty, dryRun) + else: + return update_object(obj, data, includeOnly, omit, updateEmpty, dryRun) + +def create_object(context, type_name, data, + includeOnly=None, omit=[], updateEmpty=[], dryRun=False): + logCreate = data.get('_log_create', True) + ident = data.get('_identifier') + if logCreate: + logger.info('create_object %s %s: %s' % (type_name, ident, data)) + changes = collect_changes(None, data, includeOnly, omit, updateEmpty) + msg = {'action': 'create', 'identifier': ident, 'changes': changes} + if dryRun: + return msg + type = adapted(context['concepts'][type_name]) + cont = type.conceptManager or 'concepts' + name = (type.namePrefix or (type_name + '.')) + ident + attrs = {} + for attr, (ov, val) in changes.items(): + if isinstance(val, Error): + logger.warn('create_object error: %s: %s %s' % (ident, attr, val)) + msg['info'] = 'error' + return msg + attrs[attr] = val + obj = addAndConfigureObject(context[cont], Concept, name, + conceptType=baseObject(type), **attrs) + msg['uid'] = util.getUidForObject(obj) + return msg + +def update_object(obj, data, includeOnly=None, omit=[], updateEmpty=[], dryRun=False): + logUpdate = data.get('_log_update', True) + ident = (data.get('_identifier') or + getattr(obj, 'identifier', getName(baseObject(obj)))) + changes = collect_changes(obj, data, includeOnly, omit, updateEmpty) + msg = {'action': 'update', 'identifier': ident, 'uid': obj.uid, 'changes': changes} + if logUpdate and changes: + logger.info('update_object %s: %s' % (ident, changes)) + if dryRun: + return msg + for attr, (ov, nv) in changes.items(): + if isinstance(nv, Error): + logger.warn('update_object error: %s: %s %s' % (ident, attr, nv)) + msg['info'] = 'error' + return msg + setattr(obj, attr, nv) + if changes: + notify(ObjectModifiedEvent(baseObject(obj))) + return msg + +def get_object(context, type_name, data): + if context is None: + logger.error('get_object %s: context not set, data: %s' % (type_name, data)) + ident = data.get('_identifier') + if ident is None: + logger.warn('get_object %s: _identifier missing: %s' % (type_name, data)) + type = adapted(context['concepts'][type_name]) + cont = type.conceptManager or 'concepts' + name = (type.namePrefix or (type_name + '.')) + ident + ob = context[cont].get(name) + if ob: + return adapted(ob) + return None + diff --git a/cco/processor/transformer.py b/cco/processor/transformer.py new file mode 100644 index 0000000..e8c2bd8 --- /dev/null +++ b/cco/processor/transformer.py @@ -0,0 +1,93 @@ +# +# cco.processor.transformer +# + +""" +Transform a data structure (represented by a dictionary ) +""" + +from datetime import date +from logging import getLogger +import time +import traceback + +from cco.processor.common import Error, error, _invalid, _not_found + +logger = getLogger('cco.processor.transformer') + + +def transform(sdata, fmap, context=None): + tdata = {} + for tname, spec in fmap.items(): + if spec is None: + tdata[tname] = None + continue + if isinstance(spec, str): + sname, modif = spec, None + else: + sname, modif = spec + if sname is None: + if modif is None: + tdata[tname] = None + continue + svalue = sdata # modif will extract values! + else: + svalue = sdata.get(sname, _not_found) + if svalue is _not_found: + #tdata[tname] = error('transform: not found: %s' % sname) + #continue + svalue = None + if modif is None: + tvalue = svalue + else: + tvalue = modify_value(modif, svalue, context) + tdata[tname] = tvalue + return tdata + +def modify_value(fct, value, context=None): + try: + return fct(value, context) + except TypeError: + try: + return fct(value) + except: + tb = traceback.format_exc() + return error('modify_value: %s, %s\n%s' % (fct, value, tb)) + +# predefined modifiers + +def map_value(map): + def do_map(key): + v = map.get(key, _not_found) + if v is _not_found: + v = map.get('*', _not_found) + if v is _not_found: + return error('map_value: not found: %s' % key) + return v + return do_map + +def const(val): + return lambda x: val + +def int_inv(val): + if val is None: + return None + if isinstance(val, basestring) and val == '': + return _invalid + return int(val) + +def float_inv(val): + if val is None: + return None + if isinstance(val, basestring) and val == '': + return _invalid + return float(val) + +def iso_date(val, context=None, format='%Y-%m-%d'): + if val is None: + return None + if isinstance(val, date): + return val + if isinstance(val, basestring) and val == '': + return _invalid + return date(*(time.strptime(val[:10], format)[:3])) diff --git a/cco/webapi/README.rst b/cco/webapi/README.rst new file mode 100644 index 0000000..3337bfa --- /dev/null +++ b/cco/webapi/README.rst @@ -0,0 +1,207 @@ + +cco.webapi - cyberconcepts.org: Web API = REST + JSON +===================================================== + +Let's first do some common imports and initializations. + + >>> from zope.publisher.browser import TestRequest + >>> from zope.traversing.api import getName + + >>> from logging import getLogger + >>> log = getLogger('cco.webapi') + + >>> from cco.webapi.node import ApiNode + >>> from cco.webapi.tests import callPath, traverse + + >>> from loops.setup import addAndConfigureObject, addObject + >>> from loops.concept import Concept + + >>> concepts = loopsRoot['concepts'] + >>> type_type = concepts['type'] + >>> type_topic = addAndConfigureObject(concepts, Concept, 'topic', + ... conceptType=type_type) + >>> type_task = addAndConfigureObject(concepts, Concept, 'task', + ... conceptType=type_type) + >>> home = loopsRoot['views']['home'] + +We now create the first basic objects. + + >>> apiRoot = addAndConfigureObject(home, ApiNode, 'webapi') + >>> node_topics = addAndConfigureObject(apiRoot, ApiNode, 'topics') + +Querying the database with the GET method +----------------------------------------- + +We start with calling the API view of the top-level (root) API node. + + >>> from cco.webapi.server import ApiHandler + >>> handler = ApiHandler(apiRoot, TestRequest()) + >>> handler() + '[{"name": "topics"}]' + +The tests module contains a shortcout for traversing a path and calling +the corresponding view. + + >>> callPath(apiRoot) + '[{"name": "topics"}]' + +What happens upon traversing a node? + + >>> callPath(apiRoot, 'topics') + '[]' + +When a node does not exist we get a 'NotFound' exception. + + >>> from zope.publisher.interfaces import NotFound + >>> try: + ... callPath(apiRoot, 'topics/loops') + ... except NotFound: + ... print('NotFound error!') + NotFound error! + +Maybe we should assign a target: we use the topic type as target +and create a 'loops' topic. + + >>> node_topics.target = type_topic + >>> topic_loops = addAndConfigureObject(concepts, Concept, 'loops', + ... conceptType=type_topic) + +We now get a list of the target object's children. + + >>> callPath(apiRoot, 'topics') + '[{"name": "loops", "title": ""}]' + +We can also directly access the target's children using their name. + + >>> callPath(apiRoot, 'topics/loops') + '{"name": "loops", "title": ""}' + +We can also use the type hierarchy as starting point of our +journey. + + >>> node_types = addAndConfigureObject(apiRoot, ApiNode, 'types') + >>> node_types.target = type_type + + >>> callPath(apiRoot, 'types') + '[{"name": "topic", "title": ""}, ... {"name": "type", "title": "Type"}]' + + >>> callPath(apiRoot, 'types/topic') + '[{"name": "loops", "title": ""}]' + + >>> callPath(apiRoot, 'types/topic/loops') + '{"name": "loops", "title": ""}' + +Next steps +- return properties of target object as given by interface/schema +- traverse special attributes/methods (e.g. _children) of target topic + +Creating new objects with POST +------------------------------ + + >>> input = '{"name": "rdf", "title": "RDF"}' + >>> callPath(apiRoot, 'types/topic', 'POST', input=input) + INFO: Input Data: {'name': 'rdf', 'title': 'RDF'} + '{"info": "Done"}' + + >>> callPath(apiRoot, 'types/topic') + '[{"name": "loops", "title": ""}, {"name": "rdf", "title": "RDF"}]' + + >>> callPath(apiRoot, 'types/topic/rdf') + '{"name": "rdf", "title": "RDF"}' + + >>> input = '{"name": "task0001", "title": "Document loops WebAPI"}' + >>> callPath(apiRoot, 'types/task', 'POST', input=input) + INFO: Input Data: {'name': 'task0001', 'title': 'Document loops WebAPI'} + '{"info": "Done"}' + + >>> callPath(apiRoot, 'types/task') + '[{"name": "task0001", "title": "Document loops WebAPI"}]' + +Updating objects with PUT +------------------------- + + >>> input = '{"title": "loops"}' + >>> callPath(apiRoot, 'topics/loops', 'PUT', input=input) + INFO: Input Data: {'title': 'loops'} + '{"info": "Done"}' + + >>> callPath(apiRoot, 'topics') + '[{"name": "loops", "title": "loops"}, {"name": "rdf", "title": "RDF"}]' + + >>> callPath(apiRoot, 'topics/loops') + '{"name": "loops", "title": "loops"}' + +Let's just see what happens if we do not supply input data. + + >>> callPath(apiRoot, 'topics/loops', 'PUT', input='{}') + INFO: Input Data: {} + ERROR: Missing data + '{"message": "Missing data", "status": 500}' + +Create relationships (links) between objects - assign a child. + +... TODO ... + +Client module +============= + + >>> from cco.webapi.client import postMessage + + >>> postMessage('test://localhost:8123/webapi', + ... 'demo', 'query', 'topics', 'rdf') + request: POST test://localhost:8123/webapi/demo/query/topics/rdf + None + auth: None + '{"state": "success"}' + +Asynchronous processing of integrator messages +============================================== + +query action +------------ + + >>> node_domain = addAndConfigureObject(apiRoot, ApiNode, 'demo') + >>> node_query = addAndConfigureObject(node_domain, ApiNode, 'query') + >>> node_query.target = type_type + >>> node_query.viewName = 'api_integrator_query' + + >>> callPath(apiRoot, 'demo/query/topic') + request: POST test://localhost:8123/webapi/demo/list/topic + {"_item": "loops", "title": "loops"}... + auth: None + '"{\\"state\\": \\"success\\"}"' + + >>> callPath(apiRoot, 'demo/query/topic/loops') + request: POST test://localhost:8123/webapi/demo/data/topic/loops + {"title": "loops"} + auth: None + '"{\\"state\\": \\"success\\"}"' + +data action +----------- + + >>> node_data = addAndConfigureObject(node_domain, ApiNode, 'data') + >>> node_data.target = type_type + + >>> input = '{"name": "typescript", "title": "Typescript"}' + >>> callPath(apiRoot, 'demo/data/topic', 'POST', input=input) + INFO: Input Data: {'name': 'typescript', 'title': 'Typescript'} + '{"info": "Done"}' + + >>> input = '{"title": "TypeScript"}' + >>> callPath(apiRoot, 'demo/data/topic/typescript', 'POST', input=input) + INFO: Input Data: {'title': 'TypeScript'} + '{"info": "Done"}' + + >>> input = '{"title": "TypeScript"}' + >>> callPath(apiRoot, 'demo/data/topic/typescript', 'POST', input=input) + INFO: Input Data: {'title': 'TypeScript'} + '{"info": "Done"}' + + >>> callPath(apiRoot, 'demo/query/topic') + request: POST test://localhost:8123/webapi/demo/list/topic + {"_item": "loops", "title": "loops"}... + auth: None + '"{\\"state\\": \\"success\\"}"' + + diff --git a/cco/webapi/__init__.py b/cco/webapi/__init__.py new file mode 100644 index 0000000..b5dd475 --- /dev/null +++ b/cco/webapi/__init__.py @@ -0,0 +1,8 @@ +""" package cco.webapi +""" + +from cybertools.util.jeep import Jeep + +# override in your application package: +config = Jeep(integrator=dict(url='http://localhost:8123/webapi')) + diff --git a/cco/webapi/browser/__init__.py b/cco/webapi/browser/__init__.py new file mode 100644 index 0000000..61bcc01 --- /dev/null +++ b/cco/webapi/browser/__init__.py @@ -0,0 +1 @@ +# cco.webapi.browser \ No newline at end of file diff --git a/cco/webapi/browser/add.pt b/cco/webapi/browser/add.pt new file mode 100644 index 0000000..7649331 --- /dev/null +++ b/cco/webapi/browser/add.pt @@ -0,0 +1,73 @@ + + +
+ +
+ +
+ +
+ +

Edit something

+ +

+ +

+ There are 6 input errors. +

+ +
+
+ +
+
Extra top
+
+
+ +
+ +
+ +
+
Extra bottom
+
+
+ +
+ +
+

+
+
+ +   Object Name   + + + +
+
+ +
+
+ +
+ +
+ +
+ + + diff --git a/cco/webapi/browser/edit.pt b/cco/webapi/browser/edit.pt new file mode 100644 index 0000000..de78163 --- /dev/null +++ b/cco/webapi/browser/edit.pt @@ -0,0 +1,90 @@ + + + + + + + + + + + +
+
+ +
+ + + + + + +
+ +

+ Edit something +

+ +

+ +

+ There are 6 input errors. +

+ +
+
+ +
+
Extra top
+
+
+ +
+ +
+ +
+
Extra bottom
+
+
+
+
+ +
+
+ +
+
+
+
+ +
+ + + +
+ +
+ +
+ + + diff --git a/cco/webapi/client.py b/cco/webapi/client.py new file mode 100644 index 0000000..e7e6920 --- /dev/null +++ b/cco/webapi/client.py @@ -0,0 +1,69 @@ +# +# cco.webapi.client +# + +""" +Functions for providieng external services with object data +via a REST-JSON API. +""" + +import json +from logging import getLogger +import requests + +from cco.processor import hook +from cco.webapi import testing + +logger = getLogger('cco.webapi.client') + + +def sendJson(url, payload, cred, method): + if url.startswith('test:'): + resp = testing.request(method, url, json=payload, auth=cred) + else: + if isinstance(payload, basestring): + resp = requests.request( + method, url, data=payload, auth=cred, timeout=10) + else: + resp = requests.request( + method, url, json=payload, auth=cred, timeout=10) + logger.info('sendJson: %s %s -> %s %s.' % ( + method, url, resp.status_code, resp.text)) + # TODO: check resp.status_code + #return resp.json(), dict(state='success') + return resp.content + + +def postJson(url, payload, cred): + return sendJson(url, payload, cred, 'POST') + + +def postMessage(baseUrl, domain='system', action='data', class_='', item='', + payload=None, cred=None): + url = '/'.join(p for p in (baseUrl, domain, action, class_, item) if p) + return postJson(url, payload, cred) + + +def postStandardMessage(action='data', class_="", item='', payload=None): + from cco.webapi import config + baseUrl = config.integrator.get('url') or 'http://localhost:8123' + domain = config.integrator.get('domain') or 'demo' + cred = config.integrator.get('cred') + return postMessage(baseUrl, domain, action, class_, item, payload, cred) + + +def notify(obj, data): + name = 'notifier' + config = obj._hook_config.get(name) + if config is None: + logger.warn('config missing: %s' % + dict(hook=name, obj=obj)) + return + baseUrl = config.get('url', 'http://localhost:8123') + cred = config.get('_credentials', ('dummy', 'dummy')) + url = '/'.join((baseUrl, obj._hook_message_base, obj.identifier)) + logger.info('notify: %s - %s - %s.' % (url, data, cred)) + postJson(url, data, cred) + + +hook.processor_hooks['notifier'] = notify diff --git a/cco/webapi/configure.zcml b/cco/webapi/configure.zcml new file mode 100644 index 0000000..952a282 --- /dev/null +++ b/cco/webapi/configure.zcml @@ -0,0 +1,151 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cco/webapi/interfaces.py b/cco/webapi/interfaces.py new file mode 100644 index 0000000..aec95a3 --- /dev/null +++ b/cco/webapi/interfaces.py @@ -0,0 +1,29 @@ +# +# +# + +""" +API node interfaces. +""" + +from zope.app.container.constraints import contains, containers +from zope.interface import Interface +from zope import schema + +from loops.interfaces import INodeSchema, INode, IViewManager + + +class IApiBase(INodeSchema): + + pass + + +class IApiNode(IApiBase, INode): + + contains(IApiBase) + + +class IApiNodeContained(Interface): + + containers(IApiNode, IViewManager) + diff --git a/cco/webapi/node.py b/cco/webapi/node.py new file mode 100644 index 0000000..1cc75a3 --- /dev/null +++ b/cco/webapi/node.py @@ -0,0 +1,15 @@ +# cco.webapi.node + +""" API node implementations. +""" + +from zope.interface import implementer + +from cco.webapi.interfaces import IApiNode, IApiNodeContained +from loops.view import Node + + +@implementer(IApiNode, IApiNodeContained) +class ApiNode(Node): + + pass diff --git a/cco/webapi/server.py b/cco/webapi/server.py new file mode 100644 index 0000000..9e11ded --- /dev/null +++ b/cco/webapi/server.py @@ -0,0 +1,341 @@ +# +# Copyright (c) 2017 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +View-like implementations for the REST API. +""" + +from datetime import date +import logging +from json import dumps, loads, JSONEncoder +from zope.app.container.traversal import ItemTraverser +from zope.cachedescriptors.property import Lazy +from zope import component +from zope.traversing.api import getName, getParent + +from cybertools.util import format +from loops.browser.concept import ConceptView +from loops.browser.node import NodeView +from loops.common import adapted, baseObject +from loops.concept import Concept +from loops.setup import addAndConfigureObject +from cco.webapi.client import postStandardMessage + +# provide lower-level (RDF-like?) API for accessing the concept map +# in a simple and generic way. +# next steps: RDF-like API for resources and tracks + + +logger = logging.getLogger('cco.webapi.server') +logger.setLevel(logging.INFO) +#logger.setLevel(logging.DEBUG) + + +class Encoder(JSONEncoder): + + def default(self, obj): + if isinstance(obj, date): + return format.formatDate(obj) + try: + return JSONEncoder.default(self, obj) + except TypeError: + id = getattr(obj, 'identifier', None) + if id is None: + return repr(obj) + else: + return id + +def dumps(obj): + return Encoder().encode(obj) + + +class ApiCommon(object): + """ Common routines for logging, error handling, etc + """ + + # HTTP Status: see https://en.wikipedia.org/wiki/List_of_HTTP_status_codes + + logger = logger + + def logInfo(self, message=None): + self.logger.info(message) + + def logDebug(self, message=None): + self.logger.debug(message) + + def success(self, message='Done', **kw): + if isinstance(message, dict): + kw.update(message) + else: + kw.update(dict(info=message)) + self.logger.debug(message) + return dumps(kw) + + def error(self, message, status=500): + self.logger.error(message) + self.request.response.setStatus(status, message) + return dumps(dict(message=message, status=status)) + + +class ApiHandler(ApiCommon, NodeView): + + def __call__(self, *args, **kw): + self.logDebug('Request Env: ' + str(self.request._environ)) + self.request.response.setHeader('content-type', 'application/json') + targetView = self.viewAnnotations.get('targetView') + if targetView is not None: + return targetView() + target = self.context.target + if target is not None: + targetView = self.getContainerView(target) + return targetView() + # TODO: check for request.method? + if self.request.method in ('PUT', 'POST'): + return self.error('Method not allowed', 405) + return dumps(self.getData()) + + PUT = __call__ + + def getData(self): + return [dict(name=getName(n)) for n in self.context.values()] + + def get(self, name): + #self.logInfo('*** NodeView: traversing ' + name) + targetView = self.viewAnnotations.get('targetView') + if targetView is not None: + #cv = self.getContainerView(targetView.adapted) + #if cv is None: + targetView = targetView.getView(name) + #else: + # targetView = cv.getView(name) + else: + target = self.context.target + if target is None: + return None + cv = self.getContainerView(target) + targetView = cv.getView(name) + if targetView is None: + return None + self.viewAnnotations['targetView'] = targetView + return self.context + + def getContainerView(self, target): + viewName = self.context.viewName or 'api_container' + return component.queryMultiAdapter( + (adapted(target), self.request), name=viewName) + + +class ApiTraverser(ItemTraverser): + + def publishTraverse(self, request, name): + if self.context.get(name) is None: + obj = ApiHandler(self.context, request).get(name) + if obj is not None: + return obj + return self.defaultTraverse(request, name) + + def defaultTraverse(self, request, name): + return super(ApiTraverser, self).publishTraverse(request, name) + + +# target / concept views + +class TargetBase(ApiCommon, ConceptView): + + routes = {} + + inputFieldMap = {} + + @Lazy + def outputFieldMap(self): + return dict((v, k) for k, v in self.inputFieldMap.items()) + + def __call__(self, *args, **kw): + if self.request.method == 'POST': + return self.create() + if self.request.method == 'PUT': + return self.update() + return dumps(self.getData()) + + def getName(self, obj): + obj = baseObject(obj) + name = getName(obj) + prefix = adapted(obj.getType()).namePrefix + if prefix and name.startswith(prefix): + name = name[len(prefix):] + return name + + def create(self): + # error + return self.error('Not allowed', 405) + + def update(self): + data = self.getInputData() + if not data: + return self.error('Missing data') + for k, v in data.items(): + setattr(self.adapted, k, v) + return self.success() + + def getInputData(self): + data = self.getPostData() + if data: + self.mapInputFieldNames(data) + self.unmarshalValues(data) + self.logInfo('Input Data: ' + repr(data)) + return data + + def getPostData(self): + stream = self.request.bodyStream + if stream is not None: + #json = stream.read(None) + json = stream.read(-1) + self.logDebug('POST Data: ' + repr(json)) + if json: + return loads(json) + return {} + + def mapInputFieldNames(self, data): + for k1 in data: + k2 = self.inputFieldMap.get(k1) + if k2 is not None: + data[k2] = data[k1] + del data[k1] + + def mapOutputFieldNames(self, data): + for k1 in data: + k2 = self.outputFieldMap.get(k1) + if k2 is not None: + data[k2] = data[k1] + del data[k1] + + def unmarshalValues(self, data): + pass + + def marshalValues(self, data): + pass + + +class TargetHandler(TargetBase): + + def create(self): + return self.update() + + def getView(self, name): + cname = self.routes.get(name) + if cname is not None: + name = cname + view = component.getMultiAdapter( + (self.adapted, self.request), name=name) + return view + + def getData(self): + obj = self.context + # TODO: use schema (self.adapted and typeInterface) to get all properties + data = dict(name=self.getName(obj), title=obj.title) + self.marshalValues(data) + self.mapOutputFieldNames(data) + return data + + +class ContainerHandler(TargetBase): + + itemViewName = 'api_target' + + def getData(self): + # TODO: check for real listing method and parameters + # (or produce list in the caller and use it directly as context) + lst = self.context.getChildren() + return [dict(name=self.getName(obj), title=obj.title) for obj in lst] + + def getView(self, name): + #print '*** ContainerHandler: traversing', name, self + # TODO: check for special attributes + # TODO: retrieve object from list of children + obj = self.getObject(name) + if obj is None: + return None + view = component.getMultiAdapter( + (adapted(obj), self.request), name=self.itemViewName) + return view + + def getObject(self, name): + self.error('getObject: To be implemented by subclass') + return None + + def createObject(self, tp, data=None): + if data is None: + data = self.getInputData() + if not data: + self.error('Missing data') + return None + cname = tp.conceptManager or 'concepts' + container = self.loopsRoot[cname] + prefix = tp.namePrefix or '' + name = data.get('name') + if name is None: + name = self.generateName(data) + obj = addAndConfigureObject(container, Concept, prefix + name, + title=data.get('title') or '', + conceptType=baseObject(tp)) + return adapted(obj) + + +class TypeHandler(ContainerHandler): + + def getData(self): + lst = self.adapted.typeInstances + return [dict(name=self.getName(obj), title=obj.title) for obj in lst] + + def getObject(self, name): + # TODO: use catalog query + tp = self.adapted + cname = tp.conceptManager or 'concepts' + prefix = tp.namePrefix or '' + return self.loopsRoot[cname].get(prefix + name) + + def create(self): + obj = self.createObject(self.adapted) + return self.success() + + +class IntegratorQuery(TypeHandler): + + itemViewName = 'api_integrator_class_query' + + +class IntegratorClassQuery(TypeHandler): + + itemViewName = 'api_integrator_item_query' + + def getData(self): + class_ = self.getName(self.context) + lst = self.adapted.typeInstances + data = [dumps(dict(_item=self.getName(obj), title=obj.title)) + for obj in lst] + return postStandardMessage('list', class_, payload='\n'.join(data)) + + +class IntegratorItemQuery(TargetHandler): + + def getData(self): + class_ = self.getName(self.context.getType()) + item = self.getName(self.context) + data = dict(title=self.context.title) + return postStandardMessage('data', class_, item, payload=dumps(data)) + diff --git a/cco/webapi/testing.py b/cco/webapi/testing.py new file mode 100644 index 0000000..cade8ac --- /dev/null +++ b/cco/webapi/testing.py @@ -0,0 +1,21 @@ +# +# cco.webapi.testing +# + +""" +Mock classes and functions for testing. +""" + +class Response(object): + + def __init__(self, status, content, text): + self.status_code = status + self.content = content + self.text = text + + +def request(method, url, json, auth): + print('request: %s %s\n%s\nauth: %s' % (method, url, json, auth)) + result = '{"state": "success"}' + return Response(200, result, result) + diff --git a/cco/webapi/tests.py b/cco/webapi/tests.py new file mode 100644 index 0000000..efb8573 --- /dev/null +++ b/cco/webapi/tests.py @@ -0,0 +1,100 @@ +# cco.webapi.tests + +""" Tests for the 'cco.webapi' package. +""" + +import io +import logging +import os +import sys +import unittest, doctest +from zope.app.testing.setup import placefulSetUp, placefulTearDown +from zope import component +from zope.interface import Interface +from zope.publisher.browser import TestRequest +from zope.publisher.interfaces.browser import IBrowserRequest + +from loops.interfaces import IConceptSchema, ITypeConcept +from loops.setup import importData as baseImportData +from loops.tests.setup import TestSite + +from cco.webapi.server import ApiHandler, ApiTraverser +from cco.webapi.server import TargetHandler, ContainerHandler, TypeHandler +from cco.webapi.server import IntegratorQuery, IntegratorClassQuery, IntegratorItemQuery + +import cco.webapi +cco.webapi.config.integrator['url'] = 'test://localhost:8123/webapi' + +class LogHandler(logging.StreamHandler): + + def emit(self, record): + print('%s: %s' % (record.levelname, record.msg)) + +logger = logging.getLogger('cco.webapi.server') +#logger.setLevel(logging.DEBUG) +logger.setLevel(logging.INFO) +logger.addHandler(LogHandler()) + + +def setUp(self): + site = placefulSetUp(True) + t = TestSite(site) + concepts, resources, views = t.setup() + loopsRoot = site['loops'] + self.globs['loopsRoot'] = loopsRoot + component.provideAdapter(TargetHandler, + (IConceptSchema, IBrowserRequest), Interface, name='api_target') + component.provideAdapter(ContainerHandler, + (IConceptSchema, IBrowserRequest), Interface, name='api_container') + component.provideAdapter(TypeHandler, + (ITypeConcept, IBrowserRequest), Interface, name='api_target') + component.provideAdapter(TypeHandler, + (ITypeConcept, IBrowserRequest), Interface, name='api_container') + component.provideAdapter(IntegratorQuery, + (ITypeConcept, IBrowserRequest), Interface, name='api_integrator_query') + component.provideAdapter(IntegratorClassQuery, + (ITypeConcept, IBrowserRequest), Interface, + name='api_integrator_class_query') + component.provideAdapter(IntegratorItemQuery, + (IConceptSchema, IBrowserRequest), Interface, + name='api_integrator_item_query') + + +def tearDown(self): + placefulTearDown() + + +def traverse(root, request, path): + obj = root + for name in path.split('/'): + trav = ApiTraverser(obj, request) + obj = trav.publishTraverse(request, name) + return obj + +def callPath(obj, path='', method='GET', params={}, input=''): + env = dict(REQUEST_METHOD=method) + request = TestRequest(body_instream=io.BytesIO(input.encode('UTF-8')), + environ=env, form=params) + if path: + obj = traverse(obj, request, path) + view = ApiHandler(obj, request) + return view() + + +class Test(unittest.TestCase): + "Basic tests." + + def testBasicStuff(self): + pass + + +def test_suite(): + flags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS + return unittest.TestSuite(( + unittest.makeSuite(Test), + doctest.DocFileSuite('README.rst', optionflags=flags, + setUp=setUp, tearDown=tearDown), + )) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite')