api.auth: HTTP Basic Auth working

.gitignore

@@ -8,10 +8,12 @@
 *.sublime-project
 *.sublime-workspace
 *.ropeproject
+uv.lock
 .env
 .private*
 .pytest.ini
 *#*#
 *.#*
 __pycache__
+log
 var

apidemo/__init__.py

@@ -0,0 +1 @@
+# py-scopes: apidemo

demo/__init__.py

@@ -0,0 +1,2 @@
+# py-scopes: demo
+

pyproject.toml

@@ -7,13 +7,12 @@ name = "py-scopes"
 version = "3.0.1"
 description = "Implementation of the unknown 'scopes' paradigm in Python"
 readme = "README.md"
-license = {text = "MIT"}
+license = "MIT"
+license-files = ["LICENSE"]
 keywords = ["scopes"]
 authors = [{name = "Helmut Merz", email = "helmutm@cy55.de"}]
 
-dependencies = [
-	"SQLAlchemy",
-]
+dependencies = ["SQLAlchemy", "pydantic", "pydantic-settings"]
 
 [project.optional-dependencies]
 postgres = [
@@ -21,7 +20,7 @@ postgres = [
 	"transaction",
 	"zope.sqlalchemy",
 ]
-app = [
+zope = [
 	"python-dotenv", 
 	"waitress",
 	"zope.authentication",
@@ -29,15 +28,23 @@ app = [
 	"zope.publisher", 
 	"zope.traversing",
 ]
-auth = ["pyjwt[crypto]", "cryptography", "requests"]
-test = ["zope.testrunner"]
-#test = ["pytest"]
+api = ["fastapi[standard]"]
+auth = ["authlib", "pyjwt[crypto]", "cryptography", "httpx2",
+		"pwdlib[argon2]", "requests"]
+zopetest = ["zope.testrunner"]
+apitest = ["pytest", "httpx2"]
 
 [tool.setuptools]
 packages = ["scopes"]
 
-#[tool.pytest.ini_options]
-#addopts = "-vv"
-#python_files = "test_standard.py" # default: run only `standard` tests
-# use .pytest.ini file with `python_files = test_*.py` to run all tests
+[tool.pytest.ini_options]
+addopts = ["-s"]
+testpaths = ["tests"]
+#python_files = ["test_api.py"]
+# use .pytest.ini file for local test config
+
+[tool.mypy]
+packages = ['scopes.api', 'scopes.core', 'scopes.shape', 
+			'scopes.storage', 'scopes.util']
+follow_imports = 'skip'
 

runtests.sh

@@ -1,5 +1,6 @@
 # runtests.sh
 # run all unit / doc tests
 
+#zope-testrunner --test-path=. -m !test_api $*
 zope-testrunner --test-path=. $*
 

scopes/api/__init__.py

@@ -0,0 +1 @@
+"""package scopes.api"""

scopes/api/auth/__init__.py

@@ -0,0 +1 @@
+"""package scopes.api.auth"""

scopes/api/auth/config.py

@@ -0,0 +1,17 @@
+# scopes.api.auth.config
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from typing import Any
+from scopes.storage.db.postgres import StorageFactory
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file='.env', extra='ignore') 
+    #, env_prefix='SCOPES_')
+    auth_method: str = 'oauth2'
+    jwt_key: str | None = None
+    jwt_algorithm: str = 'HS256'
+
+settings = Settings()
+
+auth_methods: dict[str, Any] = {}
+

scopes/api/auth/user.py

@@ -0,0 +1,87 @@
+# scopes.api.auth user
+
+from datetime import datetime
+from fastapi import APIRouter, Body, Depends, HTTPException
+from fastapi.security import HTTPBasic, HTTPBasicCredentials
+import jwt
+from pwdlib import PasswordHash
+from pydantic import BaseModel, Field
+from typing import Annotated, List
+
+from scopes.api import deps
+from scopes.api.auth import config
+from scopes.storage import user as st_user
+from scopes.storage.common import Storage
+
+class UserBase(BaseModel):
+    name: str
+    login: str
+    display_name: str
+    email: str | None = None
+    groups: List[str] = Field(default_factory=list)
+
+class User(UserBase):
+    uid: str | None = None
+    timeStamp: datetime | None = None
+    pw_hash: str | None = Field(exclude=True, default=None)
+
+class UserEdit(UserBase):
+    password: str | None = None
+
+    def hashPassword(self) -> str | None:
+        pw = self.password
+        return pw and hashPassword(pw) or None
+
+# password handling, authentication
+
+def dummy_auth() -> User:
+    return User(name='tester', login='tester', display_name='Tester')
+config.auth_methods['testing'] = dummy_auth
+
+pw_hash = PasswordHash.recommended()
+
+def hashPassword(pw):
+    return pw_hash.hash(pw)
+
+def verifyPassword(plain, hashed):
+    return pw_hash.verify(plain, hashed)
+
+security = HTTPBasic()
+
+type Credentials = Annotated[HTTPBasicCredentials, Depends(security)]
+
+def basicAuth(users: deps.Users, cred: Credentials) -> User:
+    tr = users.queryLast(login=cred.username)
+    users.storage.commit()
+    user = tr.toObject(User)
+    if verifyPassword(cred.password, user.pw_hash):
+        return user
+    raise HTTPException(status_code=401, detail='credentials', 
+                        headers={"WWW-Authenticate": "Basic"})
+config.auth_methods['basic'] = basicAuth
+
+def userFromToken(users: deps.Users, token: deps.AuthToken) -> User:
+    payload = jwt.decode(token, config.settings.jwt_key, 
+                         algorithms=[config.settings.jwt_algorithm])
+    userName = payload.get('sub')
+    tr = users.queryLast(name=userName)
+    users.storage.commit()
+    return tr.toObject(User)
+                       
+# routes
+
+router = APIRouter()
+
+@router.get('/users', response_model=list[User])
+async def get_users(users: deps.Users):
+    result = [tr.toObject(User) for tr in users.query()]
+    return result
+
+@router.post('/users', response_model=User)
+async def post_user(user: Annotated[UserEdit, Body()], users: deps.Users,
+                    auth_user: deps.AuthUser):
+    tr = st_user.User.fromObject(user)
+    users.save(tr)
+    #print('*** post_user', auth_user, tr)
+    return tr.toObject(User)
+

scopes/api/deps.py

@@ -0,0 +1,37 @@
+# scopes.api.deps
+
+"""Common dependencies and other definitions for Web API"""
+
+from fastapi import Depends
+from typing import Annotated
+
+from scopes import config
+from scopes.api.auth import config as auth_config
+from scopes.storage import common
+from scopes.storage import user as st_user
+
+type Storage = Annotated[common.Storage, Depends(storage)]
+type Users = Annotated[st_user.Users, Depends(users)]
+
+type AuthUser = Annotated[str, Depends(auth_user())]
+type AuthToken = Annotated[str, Depends(auth_token)]
+
+# storage
+
+def storage():
+    stf = config.StorageFactory(config.settings)
+    return stf(config.settings.dbschema)
+
+def users(storage: Storage):
+    return storage.create(st_user.Users)
+
+# auth
+
+def auth_user():
+    auth = auth_config.auth_methods.get(auth_config.settings.auth_method)
+    if auth is not None:
+        return auth
+
+def auth_token():
+    return None
+

scopes/api/main.py

@@ -0,0 +1,41 @@
+# scopes.api.main
+
+import sys
+if '' not in sys.path:
+    sys.path = [''] + sys.path
+
+from fastapi import FastAPI, Request
+from fastapi.responses import HTMLResponse
+import transaction  # type: ignore
+
+from scopes.api import tracking
+from scopes.api.auth import user
+
+# app and routes
+
+app = FastAPI()
+
+@app.middleware('http')
+async def handle_transactions(request: Request, call_next):
+    response = await call_next(request)
+    if response.status_code >= 400:
+        transaction.abort()
+    else:
+        transaction.commit()
+    return response
+
+app.include_router(tracking.router)
+app.include_router(user.router)
+
+@app.get('/')
+async def read_root():
+    return {'Hello': 'World'}
+
+@app.get('/demo')
+async def read_demo():
+    return HTMLResponse('<b>Hello World</b>')
+
+@app.get('/persons/{name}')
+async def read_person(name: str, q: str | None = None):
+    return {'name': name, 'query': q}
+    

scopes/api/tracking.py

@@ -0,0 +1,36 @@
+# scopes.api.tracking
+
+"""basic (generic) routes and views for tracks (records)"""
+
+from datetime import datetime
+from fastapi import APIRouter, Body
+from pydantic import BaseModel, Field
+from typing import Annotated, Any, Self
+
+from scopes.api import deps
+from scopes.storage import tracking
+
+class Track(BaseModel):
+    uid: str | None = None
+    head: dict[str, str]
+    data: dict[str, Any]
+    timeStamp: datetime | None = None
+
+# routes
+
+router = APIRouter()
+
+@router.get('/tracks', response_model=list[Track])
+async def get_tracks(storage: deps.Storage):
+    tracks = storage.create(tracking.Container)
+    result = [tr.asDict() for tr in tracks.query()]
+    return result
+
+@router.post('/tracks', response_model=Track)
+async def post_track(track: Annotated[Track, Body()], storage: deps.Storage):
+    tracks = storage.create(tracking.Container)
+    hd = track.head
+    tr = tracking.Track(hd['taskId'], hd['userName'], data=track.data)
+    tracks.save(tr)
+    return tr.asDict()
+

scopes/config.py

@@ -0,0 +1,30 @@
+# scopes.config.py
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from scopes.storage.db.postgres import StorageFactory
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file='.env', extra='ignore') 
+    #, env_prefix='SCOPES_')
+
+    base_url: str = 'https://test.cy7.eu'
+
+    log_file: str = 'log/scopes.log'
+    log_level: str = 'INFO'
+    log_format: str = '%(asctime)s %(levelname)s %(name)s %(message)s'
+    log_dateformat: str = '%Y-%m-%dT%H:%M:%S'
+
+    dbengine: str = 'postgresql+psycopg'
+    dbname: str = 'demo'
+    dbuser: str = 'demo'
+    dbpassword: str = 'secret'
+    dbschema: str = 'demo'
+
+settings = Settings()
+
+import logging
+
+logging.basicConfig(filename=settings.log_file, level=settings.log_level, 
+                    format=settings.log_format, datefmt=settings.log_dateformat)
+logging.getLogger('uvicorn.error').propagate = True
+

scopes/core/__init__.py

@@ -0,0 +1,4 @@
+"""package scopes.core"""
+
+from .core import *
+

scopes/core/actor.py

@@ -0,0 +1,40 @@
+# scopes.core.actor
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from typing import Any, Optional
+
+from scopes.util import keywords as k, Symbol
+
+# keywords:
+k('active full ok quit retired suspended')
+
+type Behavior = Callable[[Any], Awaitable[Behavior | None]]
+
+class Mailbox:
+
+    def __init__(self, maxsize:int = 0, status: Symbol = k.active):
+        self.queue: asyncio.Queue = asyncio.Queue(maxsize)
+        self.status: Symbol = status
+
+async def loop(q: asyncio.Queue, bhv: Behavior) -> None:
+    while True:
+        msg = await q.get()
+        if msg is k.quit: 
+            break
+        bhv = await bhv(msg) or bhv
+        q.task_done()
+
+def create(bhv: Behavior, maxsize: int = 0) -> Mailbox:
+    mb = Mailbox(maxsize)
+    asyncio.create_task(loop(mb.queue, bhv))
+    return mb
+
+async def send(mb: Mailbox, msg: Any, no_wait: bool = False) -> Symbol:
+    if mb.status is k.retired:
+        return k.retired
+    if no_wait and mb.queue.full():
+        return k.full
+    await mb.queue.put(msg)
+    return k.ok
+

scopes/core/core.py

@@ -0,0 +1,175 @@
+# scopes.core.core
+
+"""A concurrent cybernetic communication system"""
+
+from __future__ import annotations  # no check of type annotations at runtime
+import asyncio
+from collections.abc import Awaitable, Callable, Coroutine
+from dataclasses import dataclass, field
+from typing import Any, Self
+
+from scopes.core import actor
+from scopes.core.message import Message
+from scopes import util
+from scopes.util import keywords as k, Symbol, Symbols
+
+# keywords (domains, actions, classes, stages, data keys):
+k('csys env _meta',
+  'connect create created data default query set value',
+  'c00 e00 s00 op rcvr',
+  'initial active retired',
+  'address pred succ')
+
+type Categ = tuple[Symbol, Symbol]
+type Proc = Callable[[Message, Scope], Awaitable[Scope | None]]
+type Program = Callable[[Scope], Proc]
+type ProgramSpec = dict[tuple[Symbol, Symbol], Proc]
+type Syn = Callable[[Message], Awaitable[Symbol | None]]
+type SynOp = Callable[[Message], Message | None]
+
+type Handler = Callable[[Message, Scope], 
+     Awaitable[Message | None | tuple[Message | None, Scope | None]]]
+type Handlers = Handler | None | list[Handlers]
+type Actions = dict[Symbol, Handlers]
+
+def neuron(scope: Scope) -> actor.Mailbox:
+    async def process(msg: Message) -> actor.Behavior | None:
+        nonlocal scope
+        scope = await scope.proc(msg, scope) or scope
+        return process
+    return actor.create(process)
+
+def synapse(rcvr: actor.Mailbox, op: SynOp = lambda x: x) -> Syn:
+    async def syn(msg: Message) -> Symbol | None:
+        msg_new = op(msg)
+        if msg_new is not None:
+            return await actor.send(rcvr, msg_new)
+        return None
+    return syn
+
+env_dummy: actor.Mailbox = actor.Mailbox()
+
+def notify() -> Handler:
+    async def handle(msg: Message, scope: Scope) -> None:
+        await actor.send(scope.environ, msg)
+    return handle
+
+def gen_proc(actions: Actions, default: Handlers = notify(),
+             prepare: Handlers = None, finalize: Handlers = None) -> Proc:
+    async def proc(msg: Message, scope: Scope) -> Scope:
+        acts = actions.get(msg.action, actions.get(k.default, default))
+        acts = util.flatten(prepare, acts, finalize, ignore_none=True)
+        for act in acts:
+            msg = await act(msg, scope)
+            if isinstance(msg, tuple):
+                msg, new_scope = msg
+                if new_scope is not None:
+                    scope = new_scope
+            if msg is None:
+                break
+        return scope
+    return proc
+
+def gen_program(spec: ProgramSpec, default: Proc = gen_proc({})) -> Program:
+    def program(scope: Scope) -> Proc:
+        key = (scope.categ[1], scope.stage)
+        proc = spec.get(key, spec.get((key[0], k.default)))
+        if proc is None:
+            proc = spec.get((k.default, key[1]), default)
+        return proc
+    return program
+
+@dataclass
+class Scope:
+
+    value: Any = 0
+    categ: Categ = (k.csys, k.c00)
+    stage: Symbol = k.initial
+    proc: Proc = field(init=False)
+    program: Program = gen_program({})
+    syns: list[Syn | None] = field(default_factory=list)
+    environ: actor.Mailbox = env_dummy
+
+    def __post_init__(self):
+        self.set_proc()
+
+    def set_proc(self) -> Self:
+        self.proc = self.program(self)
+        return self
+
+def create_scope(env: actor.Mailbox, cat: Categ = (k.csys, k.c00), **kw) -> Scope:
+    return Scope(environ=env, categ=cat, **kw)
+
+def clone_scope(scope: Scope, **kw) -> Scope:
+    value = kw.get('value', 0)
+    categ = kw.get('categ', scope.categ)
+    stage = kw.get('stage', k.initial)
+    return Scope(environ=scope.environ, value=value, categ=categ, stage=stage, 
+                 program=scope.program, syns=[])
+
+async def create_zero(env: actor.Mailbox) -> actor.Mailbox:
+    scope = create_scope(env, program=gen_program({}, default=std_proc()))
+    zero = neuron(scope)
+    await notify_created(Message(k.csys), zero, scope)
+    return zero
+
+# predefined action handlers and Action sets
+
+def noop(no_forward: bool = False) -> Handler:
+    async def handle(msg: Message, scope: Scope) -> Message | None:
+        return None if no_forward else msg
+    return handle
+
+def forward(no_syns_notify: bool = False) -> Handler:
+    do_notify = notify() if no_syns_notify else None
+    async def handle(msg: Message, scope: Scope) -> None:
+        count = 0
+        for idx, syn in enumerate(scope.syns):
+            res = await syn(msg) if syn is not None else None
+            match res:
+                case k.retired: scope.syns[idx] = None
+                case k.ok: count += 1
+        if count == 0 and do_notify:
+            do_notify(msg, scope)
+    return handle
+
+def value_add() -> Handler:
+    async def handle(msg: Message, scope: Scope) -> tuple[Message, Scope]:
+        return msg, scope
+    return handle
+
+def create() -> Handler:
+    async def handle(msg: Message, scope: Scope) -> None:
+        new = neuron(clone_scope(scope, **msg.data))
+        await notify_created(msg, new, scope)
+        conn = msg.data.get(k.connect)
+    return handle
+
+def connect() -> Handler:
+    async def handle(msg: Message, scope: Scope) -> None:
+        return None
+    return handle
+
+def basic_actions() -> Actions:
+    acts: Actions = {
+        k.value: value_add(),
+        k.create: create(),
+        k.connect: connect()
+    }
+    return acts
+
+# predefined specialized procs
+
+def std_proc(actions: Actions | None = None, finalize: Handler = forward(), 
+             more: Actions = {}) -> Proc:
+    if actions is None:
+        actions = basic_actions()
+    actions.update(more)
+    return gen_proc(actions, finalize=finalize)
+
+# helpers
+
+async def notify_created(msg: Message, new: actor.Mailbox, scope: Scope) -> None:
+    msg = Message(msg.domain, k.created, data=msg.data | {k.new: new})
+    await notify()(msg, scope)
+

scopes/core/environ.py

@@ -0,0 +1,28 @@
+# scopes.core.environ
+
+"""Defining an environment for the communication system"""
+
+from scopes import core
+from scopes.core import actor
+from scopes.core.message import Message
+from scopes.util import keywords as k
+
+def create_env(meta_env: actor.Mailbox) -> actor.Mailbox:
+    scope = core.create_scope(meta_env, (k.env, k.c00), 
+                              program=core.gen_program({}, default=env_proc()))
+    return core.neuron(scope)
+
+def env_proc() -> core.Proc:
+    return core.gen_proc(env_actions(), default=None, finalize=core.notify())
+
+def handle_created() -> core.Handler:
+    async def handle(msg: Message, scope: core.Scope) -> Message:
+        return msg
+    return handle
+
+def env_actions() -> core.Actions:
+    return {
+        k.created: handle_created(),
+    }
+
+

scopes/core/message.py

@@ -0,0 +1,20 @@
+# scopes.core.message
+
+"""Generic message definition"""
+
+from scopes import shape
+
+class Message(shape.Record):
+
+    headFields: tuple[str, ...] = ('domain', 'action', 'class', 'item')
+    headFieldsAsKeywords = {'domain', 'action', 'class'}
+
+    customer = None
+
+    def addr(self):
+        hv = self.headValues
+        return hv[0], hv[2], hv[3]
+
+    def categ(self):
+        return self.domain, self.head['class']
+

scopes/org/task.py

@@ -8,6 +8,5 @@ from scopes.storage.concept import Concept
 
 class Task(Concept):
 
-    headFields = ['name']
     prefix = 'tsk' 
 

scopes/shape.py

@@ -0,0 +1,68 @@
+# scopes.shape
+
+"""really abstract basic data shape definitions"""
+
+from scopes.util import keywords as k
+
+class Record:
+
+    headFields: tuple[str, ...] = ()     # to be defined by subclass
+    headFieldsAsKeywords: set[str] = set()
+    defaultHeadValue = ''
+
+    def __init__(self, *headValues, data=None, **kw):
+        self.head = dict((k, self.defaultHeadValue) for k in self.headFields)
+        for k, v in kw.items():
+            self.set(k, v)
+        for ix, hv in enumerate(headValues):
+            self.set(self.headFields[ix], hv)
+        self.data = data or {}
+
+    def __getattr__(self, k):
+        raise AttributeError(k)
+
+    @property
+    def headValues(self):
+        return [self.head[k] for k in self.headFields]
+
+    def set(self, attr, value):
+        if attr in self.headFields:
+            if value is None:
+                value = self.defaultHeadValue
+            elif attr in self.headFieldsAsKeywords:
+                value = k[value]
+            self.head[attr] = value
+            setattr(self, attr, value)
+        else:
+            raise AttributeError(attr)
+
+    def update(self, data, overwrite=False):
+        if data is None:
+            return
+        if overwrite:
+            self.data = data
+        else:
+            self.data.update(data)
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__}: {self.headValues}, {self.data}>'
+
+    def asDict(self, flatten=False):
+        if flatten:
+            return self.head | self.data
+        return dict(head=self.head, data=self.data)
+
+    def toObject(self, cls):
+        return cls(**self.asDict(flatten=True))
+
+    @staticmethod
+    def prepareData(data, obj):
+        return data
+
+    @classmethod
+    def fromObject(cls, obj):
+        inp = obj.model_dump()
+        headValues = [inp.pop(hf) for hf in cls.headFields]
+        data = cls.prepareData(inp, obj)
+        return cls(*headValues, data=data)
+

scopes/storage/common.py

@@ -7,6 +7,7 @@ from sqlalchemy import create_engine, MetaData, text
 from sqlalchemy import Integer
 from sqlalchemy.dialects.sqlite import JSON
 import threading
+from typing import Any
 
 
 class Storage(object):
@@ -85,7 +86,7 @@ class StorageFactory:
         conn.commit()
 
     IdType = Integer
-    JsonType = JSON
+    JsonType: Any = JSON
 
     storageClass = Storage
 
@@ -101,7 +102,7 @@ class StorageFactory:
 
 # store information about container implementations, identified by a uid prefix.
 
-registry = {}
+registry: dict[str, Any] = {}
 
 def registerContainerClass(cls):
     prefix = cls.itemFactory.prefix

scopes/storage/concept.py

@@ -2,17 +2,17 @@
 
 """Core classes for concept map structure."""
 
-from zope.interface import implementer
+from zope.interface import implementer # type: ignore
 from scopes.interfaces import IContainer
 from scopes.storage.common import registerContainerClass, registry
-from scopes.storage.tracking import Container, Track
+from scopes.storage import tracking
 
 defaultPredicate = 'standard'
 
 
-class Concept(Track):
+class Concept(tracking.Track):
 
-    headFields = ['name']
+    headFields: tuple[str, ...] = ('name',)
 
     def parents(self, predicate=None):
         return (r.getFirst() for r in self.parentRels(predicate))
@@ -34,17 +34,17 @@ class Concept(Track):
         rels.save(Triple(self.uid, child.uid, predicate))
 
 
-class Concepts(Container):
+class Concepts(tracking.Container):
 
     insertOnChange = False
-    indexes = None
+    indexes: tracking.Indexes = None
 
     def queryRels(self, **crit):
         #pred = crit.get(predicate)
         #if pred is not None and isinstance(pred, ('string', 'bytes')):
         #    crit['predicate'] = self.storage.getContainer(Predicate).queryLast(name=pred)
         for k, v in crit.items():
-            if isinstance(v, Track):
+            if isinstance(v, tracking.Track):
                 crit[k] = v.uid
         rels = self.storage.getContainer(Triple)
         return rels.query(**crit)
@@ -69,9 +69,9 @@ def storePredicate(storage, name):
     preds.save(Predicate(name))
 
 
-class Triple(Track):
+class Triple(tracking.Track):
 
-    headFields = ['first', 'second', 'predicate']
+    headFields = ('first', 'second', 'predicate')
     prefix = 'rel'
 
     def getFirst(self):
@@ -85,7 +85,7 @@ class Triple(Track):
 
 
 @registerContainerClass
-class Rels(Container):
+class Rels(tracking.Container):
 
     itemFactory = Triple
     indexes = [('first', 'predicate', 'second'),
@@ -99,7 +99,7 @@ class Rels(Container):
 @implementer(IContainer)
 class Type(Concept):
 
-    headFields = ['name', 'tprefix']
+    headFields = ('name', 'tprefix')
     prefix = 'type'
 
     @property

scopes/storage/db/postgres.py

@@ -6,13 +6,13 @@ from sqlalchemy import create_engine
 from sqlalchemy import BigInteger
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import scoped_session, sessionmaker
-import transaction
-from zope.sqlalchemy import register, mark_changed
+import transaction # type: ignore
+from zope.sqlalchemy import register, mark_changed # type: ignore
 
-from scopes.storage.common import StorageFactory
+from scopes.storage.common import StorageFactory as BaseStorageFactory
 
 
-class StorageFactory(StorageFactory):
+class StorageFactory(BaseStorageFactory):
 
     def sessionFactory(self):
         Session = scoped_session(sessionmaker(bind=self.engine, twophase=True))

scopes/storage/folder.py

@@ -1,6 +1,6 @@
 # scopes.storage.folder
 
-from zope.interface import implementer
+from zope.interface import implementer # type: ignore
 
 from scopes.interfaces import IContainer, IReference
 from scopes.storage.common import registerContainerClass
@@ -22,7 +22,7 @@ class DummyFolder(dict):
 @implementer(IContainer, IReference)
 class Folder(Track):
 
-    headFields = ['parent', 'name', 'ref']
+    headFields = ('parent', 'name', 'ref')
     prefix = 'fldr'
 
     def values(self):

scopes/storage/message.py

@@ -2,20 +2,20 @@
 
 """Generic messages (or events) to be stored in SQL database."""
 
+from scopes.core.message import Message as BaseMessage
 from scopes.storage.common import registerContainerClass
-from scopes.storage.tracking import Container, Track
+from scopes.storage import tracking
 
 
-class Message(Track):
+class Message(BaseMessage, tracking.Track):
 
-    headFields = ['domain', 'action', 'class', 'item']
     prefix = 'msg'
 
 
 @registerContainerClass
-class Messages(Container):
+class Messages(tracking.Container):
 
-    itemFactory = Message
+    itemFactory: type[Message] = Message
     indexes = [('domain', 'action', 'class', 'item'), ('domain', 'class', 'item')]
     tableName = 'messages'
     insertOnChange = True

scopes/storage/proxy.py

@@ -6,7 +6,7 @@ in order to store attribute values in a SQL database.
 This is currently in concept and exploration state.
 """
 
-import transaction
+import transaction # type: ignore
 
 _not_found = object()
 

scopes/storage/tracking.py

@@ -12,47 +12,32 @@ from sqlalchemy import Table, Column, Index
 from sqlalchemy import DateTime, Text, func
 from sqlalchemy import and_
 
+from scopes import shape
 from scopes.storage.common import registerContainerClass
 
+type Indexes = list[tuple[str, ...]] | None
 
-class Track:
+class Record(shape.Record):
     
-    headFields = ['taskId', 'userName']
-    prefix = 'rec'
-
-    def __init__(self, *keys, data=None, timeStamp=None, trackId=None, 
+    def __init__(self, *headValues, data=None, timeStamp=None, trackId=None, 
                  container=None, **kw):
-        self.head = {}
-        for k, v in kw.items():
-            if k in self.headFields:
-                self.head[k] = kw.pop(k)
-        for ix, k in enumerate(keys):
-            self.head[self.headFields[ix]] = k
-        for k in self.headFields:
-            if self.head.get(k) is None:
-                self.head[k] = ''
-            setattr(self, k, self.head[k])
-        self.data = data or {}
+        super().__init__(*headValues, data=data, **kw)
         self.timeStamp = timeStamp
         self.trackId = trackId
         self.container = container
 
-    def set(self, attr, value):
-        if attr in self.headFields:
-            if value is None:
-                value = ''
-            self.head[attr] = value
-            setattr(self, attr, value)
-        else:
-            raise AttributeError(attr)
+    def asDict(self, flatten=False):
+        ts = str(self.timeStamp)[:19]
+        base = dict(uid=self.uid, timeStamp=ts)
+        if flatten:
+            return base | self.head | self.data
+        return base | super().asDict()
 
-    def update(self, data, overwrite=False):
-        if data is None:
-            return
-        if overwrite:
-            self.data = data
-        else:
-            self.data.update(data)
+    @staticmethod
+    def prepareData(data, obj):
+        data.pop('uid', None)
+        data.pop('timeStamp', None)
+        return data
 
     @property
     def uid(self):
@@ -66,13 +51,6 @@ class Track:
             return ''
         return str(self.trackId)
 
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self.asDict())
-
-    def asDict(self):
-        return dict(uid=self.uid, head=self.head, data=self.data, 
-                    timeStamp=str(self.timeStamp)[:19])
-
     # shortcuts for compatibility with existing software:
 
     def updateIndex(self, **kw):
@@ -83,15 +61,20 @@ class Track:
     def storeTrack(self):
         self.container.update(self)
 
+class Track(Record):
+    
+    headFields: tuple[str, ...] = ('taskId', 'userName')
+    prefix = 'rec'
+
 
 @registerContainerClass
-class Container(object):
+class Container:
 
-    itemFactory = Track
+    itemFactory: type[Track] = Track
     tableName = 'tracks'
     insertOnChange = True # always insert new track when data are changed
-    indexes = None  # default, will be overwritten by registerContainerClass()
-    #indexes = [('username',), ('taskid', 'username')] # or put explicitly in class
+    indexes: Indexes = None  # is set in registerContainerClass()
+    #indexes = [('username',), ('taskid', 'username')] # explicit setting in subclass
 
     table = None
 
@@ -137,21 +120,24 @@ class Container(object):
         changed = False
         if found.data != track.data:
             found.update(track.data)
+            track.data = found.data
             changed = True
         if track.timeStamp is not None and found.timeStamp != track.timeStamp:
             found.timeStamp = track.timeStamp
             changed = True
         if changed:
-            self.update(found)
+            self.update(found, not track.timeStamp)
         track.trackId = found.trackId
+        track.timeStamp = found.timeStamp
         return found.trackId
 
     def insert(self, track, withTrackId=False):
         t = self.table
         values = self.setupValues(track, withTrackId)
-        stmt = t.insert().values(**values).returning(t.c.trackid)
-        trackId = self.session.execute(stmt).first()[0]
+        stmt = t.insert().values(**values).returning(t.c.trackid, t.c.timestamp)
+        trackId, ts = self.session.execute(stmt).first()
         track.trackId = trackId
+        track.timeStamp = ts
         self.storage.mark_changed()
         return trackId
 

scopes/storage/user.py

@@ -0,0 +1,26 @@
+# scopes.storage.user
+
+from scopes.storage.common import registerContainerClass
+from scopes.storage import concept
+
+class User(concept.Concept):
+
+    headFields = ('name', 'login')
+    prefix = 'user'
+
+    @staticmethod
+    def prepareData(data, obj):
+        data = concept.Concept.prepareData(data, obj)
+        pw = data.pop('password', None)
+        if pw:
+            data['pw_hash'] = obj.hashPassword()
+        return data
+
+
+@registerContainerClass
+class Users(concept.Concepts):
+
+    itemFactory = User
+    indexes = [('name',), ('login',)]
+    tableName = 'users'
+

scopes/testing.py

@@ -0,0 +1,4 @@
+# scopes.testing
+
+"""Common stuff (config, fixtures, ...) for testing"""
+

scopes/tests/config.py

@@ -1,29 +1,41 @@
 # py-scopes/tests/config.py
 
 import logging
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file='.env')
+
+    base_url: str = 'https://test.cy7.eu'
+
+    log_file: str = 'scopes/tests/log/scopes-test.log'
+    log_level: str = 'INFO'
+    log_format: str = '%(asctime)s %(levelname)s %(name)s %(message)s'
+    log_dateformat: str = '%Y-%m-%dT%H:%M:%S'
+
+
+    dbengine: str = 'postgresql+psycopg'
+    dbname: str = 'testdb'
+    dbuser: str = 'testuser'
+    dbpassword: str = 'secret'
+    dbschema: str = 'testing'
+
+settings = Settings()
+
 from os import getenv
 import sys
 
-#from scopes.web.app import demo_app, zope_app
+base_url = settings.base_url
 
-log_file = 'scopes/tests/log/scopes-test.log'
-log_level = logging.INFO
-log_format = '%(asctime)s %(levelname)s %(name)s %(message)s'
-log_dateformat = '%Y-%m-%dT%H:%M:%S'
+logging.basicConfig(filename=settings.log_file, level=settings.log_level, 
+                    format=settings.log_format, datefmt=settings.log_dateformat)
 
-# server / app settings
-server_port = '8999'
-base_url = 'testing:'
-#app = zope_app
-
-# storage settings
-
-# SQLite
-dbengine = 'sqlite'
-dbname = 'var/test.db'
-dbuser = None
-dbpassword = None
-dbschema = None
+# SQLite storage settings
+dbengine = settings.dbengine
+dbname = settings.dbname
+dbuser = settings.dbuser
+dbpassword = settings.dbpassword
+dbschema = settings.dbschema
 
 # special testing stuff
 from scopes.tests import data_auth # add oidc URIs and keys to dummy_requests data

scopes/tests/test_postgres.py

@@ -6,7 +6,7 @@ import os, sys
 sys.path = [os.path.dirname(__file__)] + sys.path
 
 import unittest
-from scopes.tests import tlib_storage
+from scopes.tests import tlib_storage, tlib_web
 
 from scopes.tests.util import setup_logging
 from scopes.storage.db.postgres import StorageFactory 
@@ -38,6 +38,12 @@ class Test(unittest.TestCase):
     def test_005_message(self):
         tlib_storage.test_message(self, config)
 
+    def test_013_web(self):
+        tlib_web.test_app(self, config)
+        tlib_web.test_auth(self, config)
+        tlib_web.test_user_data(self, config)
+
+
 def test_suite():
     return unittest.TestSuite((
         unittest.TestLoader().loadTestsFromTestCase(Test),

scopes/tests/tlib_storage.py

@@ -4,6 +4,10 @@
 
 from datetime import datetime
 from scopes.storage import concept, folder, message, topic, tracking
+from scopes.util import keywords as k
+
+# predefine keywords used as domain, action, class head values in messages
+k('session system')
 
 
 def test_tracking(self, config):

scopes/tests/tlib_web.py

@@ -42,7 +42,7 @@ def test_auth(self, config):
     logger.info('test_auth keys: %s', keys)
 
 def test_user_data(self, config):
-    from scopes.org import user
+    from scopes.web.auth import user
     u1 = user.User(login='tst9', email='tst9@example.com',
                    firstName='Test', lastName='User')
     xu1 = user.ExtUser(u1, idPrefix='test.')

scopes/tests/util.py

@@ -3,9 +3,10 @@
 import logging
 
 def setup_logging(config):
+    settings = config.settings
     hdlr = logging.getLogger().handlers[-1]
     logging.getLogger().removeHandler(hdlr) # remove NullHandler added by testrunner
-    logging.basicConfig(filename=config.log_file, level=config.log_level, 
-                        format=config.log_format, datefmt=config.log_dateformat)
+    logging.basicConfig(filename=settings.log_file, level=settings.log_level, 
+                        format=settings.log_format, datefmt=settings.log_dateformat)
 
 

scopes/util.py

@@ -1,10 +1,53 @@
 # scopes.util
 
 import base64
+from collections.abc import Iterable
 import hashlib
 from secrets import choice
 import string
 
+# symbols and keywords
+
+class Symbol(str):
+
+    def __repr__(self):
+        return f'<{self}>'
+
+class Symbols:
+
+    def __init__(self, ns=None):
+        self.__prefix = ns + '.' if ns else ''
+
+    def __call__(self, *keys):
+        for segment in keys:
+            for key in segment.split():
+                if not hasattr(self, key):
+                    setattr(self, key, Symbol(self.__prefix + key))
+        return self
+
+    def __getattr__(self, k):
+        raise AttributeError(k)
+
+    def __getitem__(self, k):
+        return getattr(self, k)
+
+keywords = Symbols()
+
+# list / iterable utilities
+
+def flatten(*it, ignore_none=False):
+    accu = []
+    def doit(it):
+        if ignore_none and it is None:
+            return
+        if isinstance(it, Iterable):
+            for t in it:
+                doit(t)
+        else:
+            accu.append(it)
+    doit(it)
+    return accu
+
 # random strings, hashes, encodings
 # for authentication, encryption, and other purposes
 

scopes/web/auth/user.py

@@ -2,6 +2,9 @@
 
 """Basic user account (principal) definitions + access to identity provider."""
 
+# should be moved to scopes.web.auth
+# scopes.org: + person
+
 from dataclasses import dataclass, field
 from typing import List, Optional
 

tests/test.env

@@ -0,0 +1,24 @@
+# tests/test.env
+
+LOG_FILE=tests/log/scopes-test.log
+
+BASE_URL=http://test
+
+DBNAME=testdb
+DBUSER=testuser
+DBPASSWORD=secret
+DBSCHEMA=testing
+
+AUTH_METHOD=basic
+
+# openssl rand -hex 32:
+JWT_KEY=391a521ee5ba2b6ce67b8fdbb9f0ca88f40a87d3bc1e82974381b647bb0fc776
+JWT_ALGORITHM=HS256
+
+OIDC_PROVIDER=http://oidc.test
+OIDC_CLIENT_ID=12345
+OIDC_COOKIE_CRYPT=secret_crypt
+OIDC_ORGANIZATION_ID=12346
+OIDC_PROJECT_ID=12347
+
+

tests/test_api.py

@@ -0,0 +1,94 @@
+# py-scopes: tests.test_api
+
+"""Tests for the fastapi- (and asyncio-) based stuff"""
+
+from scopes import config
+from scopes.api.auth import config as auth_config
+config.settings = config.Settings(_env_file='tests/test.env')
+auth_config.settings = auth_config.Settings(_env_file='tests/test.env')
+
+import pytest
+from fastapi.testclient import TestClient
+from httpx2 import ASGITransport, AsyncClient, BasicAuth
+
+from scopes.api import main
+from scopes.api.main import app
+from scopes.storage import tracking, user as st_user
+from scopes.api.auth import user as api_user
+
+@pytest.fixture(scope='module')
+def storage():
+    storage = config.StorageFactory(config.settings)(config.settings.dbschema)
+    storage.dropTable('tracks')
+    storage.dropTable('users')
+    storage.commit()
+    return storage
+
+def async_client():
+    return AsyncClient(transport=ASGITransport(app=app), base_url='http://test')
+
+#@pytest.mark.anyio
+#async def test_root():
+    #async with async_client() as clt:
+        #response = await clt.get('/')
+
+def test_root():
+    with TestClient(app) as clt:
+        response = clt.get('/')
+        assert response.status_code == 200
+        assert response.json() == {'Hello': 'World'}
+
+def test_tracking(storage, caplog):
+    tracks = storage.create(tracking.Container)
+    tr01 = tracking.Track('t01', 'john')
+    tr01.update(dict(activity='testing'))
+    trid01 = tracks.save(tr01)
+    storage.commit()
+    with TestClient(app) as clt:
+        caplog.set_level(config.settings.log_level)
+        tracks = clt.get('/tracks').json()
+        assert len(tracks) == 1
+        tr = tracks[0]
+        assert tr['timeStamp'].startswith('202')
+        tr['timeStamp'] = '...'
+        assert tr == {'uid': 'rec-1', 'head': {'taskId': 't01', 'userName': 'john'}, 
+                    'data': {'activity': 'testing'}, 'timeStamp': '...'}
+        resp = clt.post('/tracks', json={'head': {'taskId': 't02', 'userName': 'jim'},
+                    'data': {'activity': 'implement post track'}})
+        assert resp.status_code == 200
+        tr02 = resp.json()
+        assert tr02['uid'] == 'rec-2'
+        #print('\n*** Logs: ', caplog.text)
+        #print('*** tr02', tr02)
+
+def test_user(storage, caplog):
+    users = storage.create(st_user.Users)
+    pw = 'secret'
+    pw_hash = api_user.hashPassword(pw)
+    u01 = st_user.User('jim', 'jim', data=dict(display_name='Jim', pw_hash=pw_hash))
+    u01id = users.save(u01)
+    storage.commit()
+    with TestClient(app) as clt:
+        caplog.set_level(config.settings.log_level)
+        auth = BasicAuth(username='jim', password='secret')
+        users = clt.get('/users').json()
+        assert len(users) == 1
+        u01i = users[0]
+        assert u01i.get(pw_hash) is None
+        inp = {'name': 'jane', 'login': 'jane', 'display_name': 'Jane', 
+               'email': 'jane@xmpl.org', 'password': 'secret'}
+        resp = clt.post('/users', json=inp, auth=auth)
+        assert resp.status_code == 200
+        u02 = resp.json()
+        assert u02['uid'] == 'user-2'
+        ts02 = u02.get('timeStamp')
+        assert ts02 is not None
+        del inp['password']
+        inp['display_name'] = 'Jane Doe'
+        resp = clt.post('/users', json=inp, auth=auth)
+        assert resp.status_code == 200
+        u02a = resp.json()
+        assert u02a['uid'] == 'user-2'
+        ts02a = u02a.get('timeStamp')
+        assert ts02a is not None
+

tests/test_config.py

@@ -0,0 +1,16 @@
+# py-scopes: tests.test_config
+
+"""Tests for scopes configuration settings via config.py and .env"""
+
+import os
+import pytest
+
+from scopes import config
+config.settings = config.Settings(_env_file='tests/test.env')
+
+from scopes.config import settings
+
+def test_config():
+    assert settings.dbname == 'testdb'
+    assert settings.dbschema == 'testing'
+

tests/test_core.py

@@ -0,0 +1,89 @@
+# py-scopes: tests.test_core
+
+""" Tests for scopes.core modules, including e.g. actor, message"""
+
+import asyncio
+import pytest
+
+from scopes import core
+from scopes.core import actor, environ
+from scopes.core.message import Message
+from scopes.util import keywords as k
+k('data', 'hello', 'person', 'show', 'switch', 'quit', 'test')
+
+@pytest.fixture
+def collector():
+    return []
+
+
+def show(msg, coll):
+    coll.append(msg)
+
+def handle_basic(coll):
+    async def handle(msg):
+        match msg:
+          case k.show: show(msg, coll)
+          case k.switch: return handle_alt(coll)
+          case _: show(msg, coll)
+    return handle
+
+def handle_alt(coll):
+    async def handle(msg):
+        match msg:
+          case k.show: show('switched', coll)
+          case _: await asyncio.sleep(0.01); show(msg, coll)
+    return handle
+
+
+@pytest.mark.anyio
+async def test_actor(collector):
+    mb1 = actor.create(handle_basic(collector))
+    mb2 = actor.create(handle_basic(collector))
+    msgs = [Message(k.test, k.hello), k.show, k.switch, k.show, "actor 1: Goodbye"]
+    await asyncio.gather(*[actor.send(mb1, m) for m in msgs])
+    await actor.send(mb2, "actor 2")
+    await actor.send(mb2, k.show)
+    await actor.send(mb2, k.quit)
+    await actor.send(mb2, k.quit)
+    await asyncio.sleep(0.01)
+    #print(collector)
+    assert len(collector) == 6
+    assert collector[-1] == 'actor 1: Goodbye'
+
+
+def test_message():
+    msg = Message(k.test, k.data, 'person', '4711', 
+                  data=dict(firstName='Bob', lastName='Miller'))
+    assert msg.head['class'] is k.person
+    assert msg.addr() == (k.test, k.person, '4711')
+    assert msg.categ() == (k.test, k.person)
+    assert msg.asDict() == {'head': {'domain': k.test, 'action': k.data, 
+                                     'class': k.person, 'item': '4711'},
+                            'data': {'firstName': 'Bob', 'lastName': 'Miller'}}
+    assert msg.asDict(flatten=True) == {
+            'domain': k.test, 'action': k.data, 'class': k.person, 'item': '4711',
+            'firstName': 'Bob', 'lastName': 'Miller'}
+
+
+@pytest.fixture
+async def meta_env(collector):
+    async def collect(msg):
+        collector.append(msg)
+    return actor.create(collect)
+
+@pytest.mark.anyio
+async def test_core(meta_env, collector):
+    env = environ.create_env(meta_env)
+    zero = await core.create_zero(env)
+    await actor.send(env, Message(k.env, k.hello))
+    await actor.send(zero, Message(k.csys, k.hello))
+    await actor.send(zero, Message(k.csys, k.create,
+                           data={k.connect: k.pred, k.address: (1, 1)}))
+    await asyncio.sleep(0.01)
+    assert len(collector) == 4
+    assert collector[0].action is k.created
+    assert collector[1].domain is k.env
+    assert collector[2].domain is k.csys
+    assert collector[3].action is k.created
+    #print(collector)
+

tests/test_storage.py

@@ -0,0 +1,130 @@
+# py-scopes: tests.test_storage
+
+""" Tests for the (PostgreSQL) storage-related functionalities"""
+
+from scopes import config
+config.settings = config.Settings(_env_file='tests/test.env')
+
+from datetime import datetime
+import pytest
+
+from scopes.storage import concept, folder, message, topic, tracking
+from scopes.util import keywords as k
+
+# predefined keywords used as domain, action, class head values in messages
+k('session system')
+
+@pytest.fixture(scope='module')
+def storage():
+    storage = config.StorageFactory(config.settings)(config.settings.dbschema)
+    storage.dropTable('tracks')
+    storage.dropTable('folders')
+    storage.dropTable('types')
+    storage.dropTable('preds')
+    storage.dropTable('rels')
+    storage.dropTable('topics')
+    storage.dropTable('messages')
+    storage.commit()
+    return storage
+
+def test_tracking(storage):
+    tracks = storage.create(tracking.Container)
+    tr01 = tracking.Track('t01', 'john')
+    assert tr01.head == {'taskId': 't01', 'userName': 'john'}
+    assert tr01.taskId == 't01'
+    assert tr01.userName == 'john'
+    tr01.update(dict(activity='testing'))
+    trid01 = tracks.save(tr01)
+    assert trid01 > 0
+
+    tr01a = tracks[f'{trid01:07}']
+    assert tr01a.head == tr01.head
+    assert tr01a.trackId == trid01
+    assert tr01a.data.get('activity') == 'testing'
+    tr01a.update(dict(text='Set up unit tests.'))
+    tr01a.timeStamp = None
+    assert tracks.save(tr01a) > 0
+
+    tr01b = tracks.queryLast(taskId='t01')
+    assert tr01b.head == tr01.head
+    assert tr01b.trackId != trid01
+    assert tr01b.data.get('activity') == 'testing'
+    assert tr01b.data.get('text') == 'Set up unit tests.'
+
+    tr02 = tracking.Track('t02', 'jim', trackId=31, timeStamp=datetime(2023, 11, 30),
+                        data=dict(activity='concept'))
+    trid02 = tracks.upsert(tr02)
+    assert trid02 == 31
+    assert tr02.uid == 'rec-31'
+    tr02.trackId = trid01
+    trid021 = tracks.upsert(tr02)
+    assert trid021 == trid01
+    assert tr02.uid == 'rec-' + str(trid01)
+
+    tr03 = storage.getItem('rec-31')
+    assert tr03.trackId == 31
+    n = tracks.remove(tr03)
+    assert n == 1
+    assert tracks.get(31) == None
+
+    storage.commit()
+
+def test_folder(storage):
+    root = folder.Root(storage)
+    assert list(root.keys()) == []
+    root['top'] = folder.Folder()
+    assert list(root.keys()) == ['top']
+    top = root['top']
+    top['child1'] = folder.Folder(data=dict(title='First Child'))
+    assert list(top.keys()) == ['child1']
+    ch1 = top['child1']
+    assert ch1.parent == top.rid
+    assert list(top.keys()) == ['child1']
+    ch1.set('name', 'level2-item1')
+    ch1.storeTrack()
+    storage.commit()
+
+def test_type(storage):
+    concept.setupCoreTypes(storage)
+    types = storage.getContainer(concept.Type)
+    tps = list(types.query())
+    assert len(tps) == 8
+    tfolder = types.queryLast(name='folder')
+    fldrs = list(tfolder.values())
+    assert len(fldrs) == 2
+    assert fldrs[0].name == 'top'
+    storage.commit()
+
+def test_topic(storage):
+    rels = storage.getContainer(concept.Triple)
+    topics = storage.getContainer(topic.Topic)
+    types = storage.getContainer(concept.Type)
+    concept.storePredicate(storage, concept.defaultPredicate)
+    root = folder.Root(storage)
+    root['top']['topics'] = ftopics = folder.Folder()
+    ttopic = types.queryLast(name='topic')
+    assert ttopic.name == 'topic'
+    ftopics.setTarget(ttopic)
+    assert ftopics.getTarget().name == 'topic'
+    tp_itc = topic.Topic('itc', data=dict(
+        title='ITC', description='Information and Communication Technology'))
+    topics.save(tp_itc)
+    tp_proglang = topic.Topic('prog_lang', data=dict(
+        title='Programming Languages', 
+        description='Programming Languages'))
+    topics.save(tp_proglang)
+    tp_itc.addChild(tp_proglang)
+    c = list(tp_itc.children())
+    assert c[0].name == 'prog_lang'
+    storage.commit()
+
+def test_message(storage):
+    messages = storage.create(message.Messages)
+    m01 = message.Message('system', 'data', 'session', 'V1_317784226621611853')
+    m01.update(dict(userid='tst9'))
+    mid01 = messages.save(m01)
+    msgs = list(messages.query())
+    assert msgs[0].domain is k.system
+    assert len(msgs) == 1
+    storage.commit()
+

tests/test_util.py

@@ -0,0 +1,30 @@
+# py-scopes: tests.test_util
+
+"""Tests for the scopes.util package"""
+
+import pytest
+
+from scopes.util import keywords as k, flatten, Symbols
+
+k('dummy other', 'cd', 'new', 'erp')
+
+dom = Symbols('dom')('crm erp sys')
+
+def test_keywords():
+    assert k.dummy == 'dummy'
+    assert k.dummy is not 'dummy'
+    assert k.dummy is k.dummy
+    assert k.dummy is not k.other
+    assert getattr(k, "abcd"[-2:]) is k.cd
+    assert k['dummy'] is k.dummy
+    assert k['new'] is k['new']
+    assert dom.erp == 'dom.erp'
+    assert dom.erp is not 'dom.erp'
+    assert repr(dom.erp) == '<dom.erp>'
+    assert dom.erp != k.erp
+
+def test_flatten():
+    lst = (1, (None, 2), (3, (4, 5)))
+    assert flatten(lst) == [1, None, 2, 3, 4, 5]
+    assert flatten(42, lst, (6, 7), ignore_none=True) == [42, 1, 2, 3, 4, 5, 6, 7]
+