cco/py-scopes
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: cco:master
Branches Tags
cco:master
cco:py3.10
cco:3.0.1-base
..
compare: cco:3.0.1-base
Branches Tags
cco:master
cco:py3.10
cco:3.0.1-base

No commits in common. "master" and "3.0.1-base" have entirely different histories.
master ... 3.0.1-base

43 changed files with 105 additions and 1171 deletions
Show stats Expand all files Collapse all files

2
.gitignore vendored
View file

@ -8,12 +8,10 @@
*.sublime-project
*.sublime-workspace
*.ropeproject
uv.lock
.env
.private*
.pytest.ini
*#*#
*.#*
__pycache__
log
var

1
apidemo/__init__.py
View file

@ -1 +0,0 @@
# py-scopes: apidemo

2
demo/__init__.py
View file

@ -1,2 +0,0 @@
# py-scopes: demo

31
pyproject.toml
View file

@ -7,12 +7,13 @@ name = "py-scopes"
version = "3.0.1"
description = "Implementation of the unknown 'scopes' paradigm in Python"
readme = "README.md"
license = "MIT"
license-files = ["LICENSE"]
license = {text = "MIT"}
keywords = ["scopes"]
authors = [{name = "Helmut Merz", email = "helmutm@cy55.de"}]
dependencies = ["SQLAlchemy", "pydantic", "pydantic-settings"]
dependencies = [
"SQLAlchemy",
]
[project.optional-dependencies]
postgres = [
@ -20,7 +21,7 @@ postgres = [
"transaction",
"zope.sqlalchemy",
]
zope = [
app = [
"python-dotenv",
"waitress",
"zope.authentication",
@ -28,23 +29,15 @@ zope = [
"zope.publisher",
"zope.traversing",
]
api = ["fastapi[standard]"]
auth = ["authlib", "pyjwt[crypto]", "cryptography", "httpx2",
"pwdlib[argon2]", "requests"]
zopetest = ["zope.testrunner"]
apitest = ["pytest", "httpx2"]
auth = ["pyjwt[crypto]", "cryptography", "requests"]
test = ["zope.testrunner"]
#test = ["pytest"]
[tool.setuptools]
packages = ["scopes"]
[tool.pytest.ini_options]
addopts = ["-s"]
testpaths = ["tests"]
#python_files = ["test_api.py"]
# use .pytest.ini file for local test config
[tool.mypy]
packages = ['scopes.api', 'scopes.core', 'scopes.shape',
'scopes.storage', 'scopes.util']
follow_imports = 'skip'
#[tool.pytest.ini_options]
#addopts = "-vv"
#python_files = "test_standard.py" # default: run only `standard` tests
# use .pytest.ini file with `python_files = test_*.py` to run all tests

1
runtests.sh
View file

@ -1,6 +1,5 @@
# runtests.sh
# run all unit / doc tests
#zope-testrunner --test-path=. -m !test_api $*
zope-testrunner --test-path=. $*

1
scopes/api/__init__.py
View file

@ -1 +0,0 @@
"""package scopes.api"""

1
scopes/api/auth/__init__.py
View file

@ -1 +0,0 @@
"""package scopes.api.auth"""

17
scopes/api/auth/config.py
View file

@ -1,17 +0,0 @@
# scopes.api.auth.config
from pydantic_settings import BaseSettings, SettingsConfigDict
from typing import Any
from scopes.storage.db.postgres import StorageFactory
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_file='.env', extra='ignore')
#, env_prefix='SCOPES_')
auth_method: str = 'oauth2'
jwt_key: str | None = None
jwt_algorithm: str = 'HS256'
settings = Settings()
auth_methods: dict[str, Any] = {}

87
scopes/api/auth/user.py
View file

@ -1,87 +0,0 @@
# scopes.api.auth user
from datetime import datetime
from fastapi import APIRouter, Body, Depends, HTTPException
from fastapi.security import HTTPBasic, HTTPBasicCredentials
import jwt
from pwdlib import PasswordHash
from pydantic import BaseModel, Field
from typing import Annotated, List
from scopes.api import deps
from scopes.api.auth import config
from scopes.storage import user as st_user
from scopes.storage.common import Storage
class UserBase(BaseModel):
name: str
login: str
display_name: str
email: str | None = None
groups: List[str] = Field(default_factory=list)
class User(UserBase):
uid: str | None = None
timeStamp: datetime | None = None
pw_hash: str | None = Field(exclude=True, default=None)
class UserEdit(UserBase):
password: str | None = None
def hashPassword(self) -> str | None:
pw = self.password
return pw and hashPassword(pw) or None
# password handling, authentication
def dummy_auth() -> User:
return User(name='tester', login='tester', display_name='Tester')
config.auth_methods['testing'] = dummy_auth
pw_hash = PasswordHash.recommended()
def hashPassword(pw):
return pw_hash.hash(pw)
def verifyPassword(plain, hashed):
return pw_hash.verify(plain, hashed)
security = HTTPBasic()
type Credentials = Annotated[HTTPBasicCredentials, Depends(security)]
def basicAuth(users: deps.Users, cred: Credentials) -> User:
tr = users.queryLast(login=cred.username)
users.storage.commit()
user = tr.toObject(User)
if verifyPassword(cred.password, user.pw_hash):
return user
raise HTTPException(status_code=401, detail='credentials',
headers={"WWW-Authenticate": "Basic"})
config.auth_methods['basic'] = basicAuth
def userFromToken(users: deps.Users, token: deps.AuthToken) -> User:
payload = jwt.decode(token, config.settings.jwt_key,
algorithms=[config.settings.jwt_algorithm])
userName = payload.get('sub')
tr = users.queryLast(name=userName)
users.storage.commit()
return tr.toObject(User)
# routes
router = APIRouter()
@router.get('/users', response_model=list[User])
async def get_users(users: deps.Users):
result = [tr.toObject(User) for tr in users.query()]
return result
@router.post('/users', response_model=User)
async def post_user(user: Annotated[UserEdit, Body()], users: deps.Users,
auth_user: deps.AuthUser):
tr = st_user.User.fromObject(user)
users.save(tr)
#print('*** post_user', auth_user, tr)
return tr.toObject(User)

37
scopes/api/deps.py
View file

@ -1,37 +0,0 @@
# scopes.api.deps
"""Common dependencies and other definitions for Web API"""
from fastapi import Depends
from typing import Annotated
from scopes import config
from scopes.api.auth import config as auth_config
from scopes.storage import common
from scopes.storage import user as st_user
type Storage = Annotated[common.Storage, Depends(storage)]
type Users = Annotated[st_user.Users, Depends(users)]
type AuthUser = Annotated[str, Depends(auth_user())]
type AuthToken = Annotated[str, Depends(auth_token)]
# storage
def storage():
stf = config.StorageFactory(config.settings)
return stf(config.settings.dbschema)
def users(storage: Storage):
return storage.create(st_user.Users)
# auth
def auth_user():
auth = auth_config.auth_methods.get(auth_config.settings.auth_method)
if auth is not None:
return auth
def auth_token():
return None

41
scopes/api/main.py
View file

@ -1,41 +0,0 @@
# scopes.api.main
import sys
if '' not in sys.path:
sys.path = [''] + sys.path
from fastapi import FastAPI, Request
from fastapi.responses import HTMLResponse
import transaction # type: ignore
from scopes.api import tracking
from scopes.api.auth import user
# app and routes
app = FastAPI()
@app.middleware('http')
async def handle_transactions(request: Request, call_next):
response = await call_next(request)
if response.status_code >= 400:
transaction.abort()
else:
transaction.commit()
return response
app.include_router(tracking.router)
app.include_router(user.router)
@app.get('/')
async def read_root():
return {'Hello': 'World'}
@app.get('/demo')
async def read_demo():
return HTMLResponse('<b>Hello World</b>')
@app.get('/persons/{name}')
async def read_person(name: str, q: str | None = None):
return {'name': name, 'query': q}

36
scopes/api/tracking.py
View file

@ -1,36 +0,0 @@
# scopes.api.tracking
"""basic (generic) routes and views for tracks (records)"""
from datetime import datetime
from fastapi import APIRouter, Body
from pydantic import BaseModel, Field
from typing import Annotated, Any, Self
from scopes.api import deps
from scopes.storage import tracking
class Track(BaseModel):
uid: str | None = None
head: dict[str, str]
data: dict[str, Any]
timeStamp: datetime | None = None
# routes
router = APIRouter()
@router.get('/tracks', response_model=list[Track])
async def get_tracks(storage: deps.Storage):
tracks = storage.create(tracking.Container)
result = [tr.asDict() for tr in tracks.query()]
return result
@router.post('/tracks', response_model=Track)
async def post_track(track: Annotated[Track, Body()], storage: deps.Storage):
tracks = storage.create(tracking.Container)
hd = track.head
tr = tracking.Track(hd['taskId'], hd['userName'], data=track.data)
tracks.save(tr)
return tr.asDict()

30
scopes/config.py
View file

@ -1,30 +0,0 @@
# scopes.config.py
from pydantic_settings import BaseSettings, SettingsConfigDict
from scopes.storage.db.postgres import StorageFactory
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_file='.env', extra='ignore')
#, env_prefix='SCOPES_')
base_url: str = 'https://test.cy7.eu'
log_file: str = 'log/scopes.log'
log_level: str = 'INFO'
log_format: str = '%(asctime)s %(levelname)s %(name)s %(message)s'
log_dateformat: str = '%Y-%m-%dT%H:%M:%S'
dbengine: str = 'postgresql+psycopg'
dbname: str = 'demo'
dbuser: str = 'demo'
dbpassword: str = 'secret'
dbschema: str = 'demo'
settings = Settings()
import logging
logging.basicConfig(filename=settings.log_file, level=settings.log_level,
format=settings.log_format, datefmt=settings.log_dateformat)
logging.getLogger('uvicorn.error').propagate = True

4
scopes/core/__init__.py
View file

@ -1,4 +0,0 @@
"""package scopes.core"""
from .core import *

40
scopes/core/actor.py
View file

@ -1,40 +0,0 @@
# scopes.core.actor
import asyncio
from collections.abc import Awaitable, Callable
from typing import Any, Optional
from scopes.util import keywords as k, Symbol
# keywords:
k('active full ok quit retired suspended')
type Behavior = Callable[[Any], Awaitable[Behavior | None]]
class Mailbox:
def __init__(self, maxsize:int = 0, status: Symbol = k.active):
self.queue: asyncio.Queue = asyncio.Queue(maxsize)
self.status: Symbol = status
async def loop(q: asyncio.Queue, bhv: Behavior) -> None:
while True:
msg = await q.get()
if msg is k.quit:
break
bhv = await bhv(msg) or bhv
q.task_done()
def create(bhv: Behavior, maxsize: int = 0) -> Mailbox:
mb = Mailbox(maxsize)
asyncio.create_task(loop(mb.queue, bhv))
return mb
async def send(mb: Mailbox, msg: Any, no_wait: bool = False) -> Symbol:
if mb.status is k.retired:
return k.retired
if no_wait and mb.queue.full():
return k.full
await mb.queue.put(msg)
return k.ok

175
scopes/core/core.py
View file

@ -1,175 +0,0 @@
# scopes.core.core
"""A concurrent cybernetic communication system"""
from __future__ import annotations # no check of type annotations at runtime
import asyncio
from collections.abc import Awaitable, Callable, Coroutine
from dataclasses import dataclass, field
from typing import Any, Self
from scopes.core import actor
from scopes.core.message import Message
from scopes import util
from scopes.util import keywords as k, Symbol, Symbols
# keywords (domains, actions, classes, stages, data keys):
k('csys env _meta',
'connect create created data default query set value',
'c00 e00 s00 op rcvr',
'initial active retired',
'address pred succ')
type Categ = tuple[Symbol, Symbol]
type Proc = Callable[[Message, Scope], Awaitable[Scope | None]]
type Program = Callable[[Scope], Proc]
type ProgramSpec = dict[tuple[Symbol, Symbol], Proc]
type Syn = Callable[[Message], Awaitable[Symbol | None]]
type SynOp = Callable[[Message], Message | None]
type Handler = Callable[[Message, Scope],
Awaitable[Message | None | tuple[Message | None, Scope | None]]]
type Handlers = Handler | None | list[Handlers]
type Actions = dict[Symbol, Handlers]
def neuron(scope: Scope) -> actor.Mailbox:
async def process(msg: Message) -> actor.Behavior | None:
nonlocal scope
scope = await scope.proc(msg, scope) or scope
return process
return actor.create(process)
def synapse(rcvr: actor.Mailbox, op: SynOp = lambda x: x) -> Syn:
async def syn(msg: Message) -> Symbol | None:
msg_new = op(msg)
if msg_new is not None:
return await actor.send(rcvr, msg_new)
return None
return syn
env_dummy: actor.Mailbox = actor.Mailbox()
def notify() -> Handler:
async def handle(msg: Message, scope: Scope) -> None:
await actor.send(scope.environ, msg)
return handle
def gen_proc(actions: Actions, default: Handlers = notify(),
prepare: Handlers = None, finalize: Handlers = None) -> Proc:
async def proc(msg: Message, scope: Scope) -> Scope:
acts = actions.get(msg.action, actions.get(k.default, default))
acts = util.flatten(prepare, acts, finalize, ignore_none=True)
for act in acts:
msg = await act(msg, scope)
if isinstance(msg, tuple):
msg, new_scope = msg
if new_scope is not None:
scope = new_scope
if msg is None:
break
return scope
return proc
def gen_program(spec: ProgramSpec, default: Proc = gen_proc({})) -> Program:
def program(scope: Scope) -> Proc:
key = (scope.categ[1], scope.stage)
proc = spec.get(key, spec.get((key[0], k.default)))
if proc is None:
proc = spec.get((k.default, key[1]), default)
return proc
return program
@dataclass
class Scope:
value: Any = 0
categ: Categ = (k.csys, k.c00)
stage: Symbol = k.initial
proc: Proc = field(init=False)
program: Program = gen_program({})
syns: list[Syn | None] = field(default_factory=list)
environ: actor.Mailbox = env_dummy
def __post_init__(self):
self.set_proc()
def set_proc(self) -> Self:
self.proc = self.program(self)
return self
def create_scope(env: actor.Mailbox, cat: Categ = (k.csys, k.c00), **kw) -> Scope:
return Scope(environ=env, categ=cat, **kw)
def clone_scope(scope: Scope, **kw) -> Scope:
value = kw.get('value', 0)
categ = kw.get('categ', scope.categ)
stage = kw.get('stage', k.initial)
return Scope(environ=scope.environ, value=value, categ=categ, stage=stage,
program=scope.program, syns=[])
async def create_zero(env: actor.Mailbox) -> actor.Mailbox:
scope = create_scope(env, program=gen_program({}, default=std_proc()))
zero = neuron(scope)
await notify_created(Message(k.csys), zero, scope)
return zero
# predefined action handlers and Action sets
def noop(no_forward: bool = False) -> Handler:
async def handle(msg: Message, scope: Scope) -> Message | None:
return None if no_forward else msg
return handle
def forward(no_syns_notify: bool = False) -> Handler:
do_notify = notify() if no_syns_notify else None
async def handle(msg: Message, scope: Scope) -> None:
count = 0
for idx, syn in enumerate(scope.syns):
res = await syn(msg) if syn is not None else None
match res:
case k.retired: scope.syns[idx] = None
case k.ok: count += 1
if count == 0 and do_notify:
do_notify(msg, scope)
return handle
def value_add() -> Handler:
async def handle(msg: Message, scope: Scope) -> tuple[Message, Scope]:
return msg, scope
return handle
def create() -> Handler:
async def handle(msg: Message, scope: Scope) -> None:
new = neuron(clone_scope(scope, **msg.data))
await notify_created(msg, new, scope)
conn = msg.data.get(k.connect)
return handle
def connect() -> Handler:
async def handle(msg: Message, scope: Scope) -> None:
return None
return handle
def basic_actions() -> Actions:
acts: Actions = {
k.value: value_add(),
k.create: create(),
k.connect: connect()
}
return acts
# predefined specialized procs
def std_proc(actions: Actions | None = None, finalize: Handler = forward(),
more: Actions = {}) -> Proc:
if actions is None:
actions = basic_actions()
actions.update(more)
return gen_proc(actions, finalize=finalize)
# helpers
async def notify_created(msg: Message, new: actor.Mailbox, scope: Scope) -> None:
msg = Message(msg.domain, k.created, data=msg.data | {k.new: new})
await notify()(msg, scope)

28
scopes/core/environ.py
View file

@ -1,28 +0,0 @@
# scopes.core.environ
"""Defining an environment for the communication system"""
from scopes import core
from scopes.core import actor
from scopes.core.message import Message
from scopes.util import keywords as k
def create_env(meta_env: actor.Mailbox) -> actor.Mailbox:
scope = core.create_scope(meta_env, (k.env, k.c00),
program=core.gen_program({}, default=env_proc()))
return core.neuron(scope)
def env_proc() -> core.Proc:
return core.gen_proc(env_actions(), default=None, finalize=core.notify())
def handle_created() -> core.Handler:
async def handle(msg: Message, scope: core.Scope) -> Message:
return msg
return handle
def env_actions() -> core.Actions:
return {
k.created: handle_created(),
}

20
scopes/core/message.py
View file

@ -1,20 +0,0 @@
# scopes.core.message
"""Generic message definition"""
from scopes import shape
class Message(shape.Record):
headFields: tuple[str, ...] = ('domain', 'action', 'class', 'item')
headFieldsAsKeywords = {'domain', 'action', 'class'}
customer = None
def addr(self):
hv = self.headValues
return hv[0], hv[2], hv[3]
def categ(self):
return self.domain, self.head['class']

1
scopes/org/task.py
View file

@ -8,5 +8,6 @@ from scopes.storage.concept import Concept
class Task(Concept):
headFields = ['name']
prefix = 'tsk'

3
scopes/web/auth/user.py → scopes/org/user.py
View file

@ -2,9 +2,6 @@
"""Basic user account (principal) definitions + access to identity provider."""
# should be moved to scopes.web.auth
# scopes.org: + person
from dataclasses import dataclass, field
from typing import List, Optional

68
scopes/shape.py
View file

@ -1,68 +0,0 @@
# scopes.shape
"""really abstract basic data shape definitions"""
from scopes.util import keywords as k
class Record:
headFields: tuple[str, ...] = () # to be defined by subclass
headFieldsAsKeywords: set[str] = set()
defaultHeadValue = ''
def __init__(self, *headValues, data=None, **kw):
self.head = dict((k, self.defaultHeadValue) for k in self.headFields)
for k, v in kw.items():
self.set(k, v)
for ix, hv in enumerate(headValues):
self.set(self.headFields[ix], hv)
self.data = data or {}
def __getattr__(self, k):
raise AttributeError(k)
@property
def headValues(self):
return [self.head[k] for k in self.headFields]
def set(self, attr, value):
if attr in self.headFields:
if value is None:
value = self.defaultHeadValue
elif attr in self.headFieldsAsKeywords:
value = k[value]
self.head[attr] = value
setattr(self, attr, value)
else:
raise AttributeError(attr)
def update(self, data, overwrite=False):
if data is None:
return
if overwrite:
self.data = data
else:
self.data.update(data)
def __repr__(self):
return f'<{self.__class__.__name__}: {self.headValues}, {self.data}>'
def asDict(self, flatten=False):
if flatten:
return self.head | self.data
return dict(head=self.head, data=self.data)
def toObject(self, cls):
return cls(**self.asDict(flatten=True))
@staticmethod
def prepareData(data, obj):
return data
@classmethod
def fromObject(cls, obj):
inp = obj.model_dump()
headValues = [inp.pop(hf) for hf in cls.headFields]
data = cls.prepareData(inp, obj)
return cls(*headValues, data=data)

5
scopes/storage/common.py
View file

@ -7,7 +7,6 @@ from sqlalchemy import create_engine, MetaData, text
from sqlalchemy import Integer
from sqlalchemy.dialects.sqlite import JSON
import threading
from typing import Any
class Storage(object):
@ -86,7 +85,7 @@ class StorageFactory:
conn.commit()
IdType = Integer
JsonType: Any = JSON
JsonType = JSON
storageClass = Storage
@ -102,7 +101,7 @@ class StorageFactory:
# store information about container implementations, identified by a uid prefix.
registry: dict[str, Any] = {}
registry = {}
def registerContainerClass(cls):
prefix = cls.itemFactory.prefix

22
scopes/storage/concept.py
View file

@ -2,17 +2,17 @@
"""Core classes for concept map structure."""
from zope.interface import implementer # type: ignore
from zope.interface import implementer
from scopes.interfaces import IContainer
from scopes.storage.common import registerContainerClass, registry
from scopes.storage import tracking
from scopes.storage.tracking import Container, Track
defaultPredicate = 'standard'
class Concept(tracking.Track):
class Concept(Track):
headFields: tuple[str, ...] = ('name',)
headFields = ['name']
def parents(self, predicate=None):
return (r.getFirst() for r in self.parentRels(predicate))
@ -34,17 +34,17 @@ class Concept(tracking.Track):
rels.save(Triple(self.uid, child.uid, predicate))
class Concepts(tracking.Container):
class Concepts(Container):
insertOnChange = False
indexes: tracking.Indexes = None
indexes = None
def queryRels(self, **crit):
#pred = crit.get(predicate)
#if pred is not None and isinstance(pred, ('string', 'bytes')):
# crit['predicate'] = self.storage.getContainer(Predicate).queryLast(name=pred)
for k, v in crit.items():
if isinstance(v, tracking.Track):
if isinstance(v, Track):
crit[k] = v.uid
rels = self.storage.getContainer(Triple)
return rels.query(**crit)
@ -69,9 +69,9 @@ def storePredicate(storage, name):
preds.save(Predicate(name))
class Triple(tracking.Track):
class Triple(Track):
headFields = ('first', 'second', 'predicate')
headFields = ['first', 'second', 'predicate']
prefix = 'rel'
def getFirst(self):
@ -85,7 +85,7 @@ class Triple(tracking.Track):
@registerContainerClass
class Rels(tracking.Container):
class Rels(Container):
itemFactory = Triple
indexes = [('first', 'predicate', 'second'),
@ -99,7 +99,7 @@ class Rels(tracking.Container):
@implementer(IContainer)
class Type(Concept):
headFields = ('name', 'tprefix')
headFields = ['name', 'tprefix']
prefix = 'type'
@property

8
scopes/storage/db/postgres.py
View file

@ -6,13 +6,13 @@ from sqlalchemy import create_engine
from sqlalchemy import BigInteger
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import scoped_session, sessionmaker
import transaction # type: ignore
from zope.sqlalchemy import register, mark_changed # type: ignore
import transaction
from zope.sqlalchemy import register, mark_changed
from scopes.storage.common import StorageFactory as BaseStorageFactory
from scopes.storage.common import StorageFactory
class StorageFactory(BaseStorageFactory):
class StorageFactory(StorageFactory):
def sessionFactory(self):
Session = scoped_session(sessionmaker(bind=self.engine, twophase=True))

4
scopes/storage/folder.py
View file

@ -1,6 +1,6 @@
# scopes.storage.folder
from zope.interface import implementer # type: ignore
from zope.interface import implementer
from scopes.interfaces import IContainer, IReference
from scopes.storage.common import registerContainerClass
@ -22,7 +22,7 @@ class DummyFolder(dict):
@implementer(IContainer, IReference)
class Folder(Track):
headFields = ('parent', 'name', 'ref')
headFields = ['parent', 'name', 'ref']
prefix = 'fldr'
def values(self):

10
scopes/storage/message.py
View file

@ -2,20 +2,20 @@
"""Generic messages (or events) to be stored in SQL database."""
from scopes.core.message import Message as BaseMessage
from scopes.storage.common import registerContainerClass
from scopes.storage import tracking
from scopes.storage.tracking import Container, Track
class Message(BaseMessage, tracking.Track):
class Message(Track):
headFields = ['domain', 'action', 'class', 'item']
prefix = 'msg'
@registerContainerClass
class Messages(tracking.Container):
class Messages(Container):
itemFactory: type[Message] = Message
itemFactory = Message
indexes = [('domain', 'action', 'class', 'item'), ('domain', 'class', 'item')]
tableName = 'messages'
insertOnChange = True

2
scopes/storage/proxy.py
View file

@ -6,7 +6,7 @@ in order to store attribute values in a SQL database.
This is currently in concept and exploration state.
"""
import transaction # type: ignore
import transaction
_not_found = object()

76
scopes/storage/tracking.py
View file

@ -12,32 +12,47 @@ from sqlalchemy import Table, Column, Index
from sqlalchemy import DateTime, Text, func
from sqlalchemy import and_
from scopes import shape
from scopes.storage.common import registerContainerClass
type Indexes = list[tuple[str, ...]] | None
class Record(shape.Record):
class Track:
def __init__(self, *headValues, data=None, timeStamp=None, trackId=None,
headFields = ['taskId', 'userName']
prefix = 'rec'
def __init__(self, *keys, data=None, timeStamp=None, trackId=None,
container=None, **kw):
super().__init__(*headValues, data=data, **kw)
self.head = {}
for k, v in kw.items():
if k in self.headFields:
self.head[k] = kw.pop(k)
for ix, k in enumerate(keys):
self.head[self.headFields[ix]] = k
for k in self.headFields:
if self.head.get(k) is None:
self.head[k] = ''
setattr(self, k, self.head[k])
self.data = data or {}
self.timeStamp = timeStamp
self.trackId = trackId
self.container = container
def asDict(self, flatten=False):
ts = str(self.timeStamp)[:19]
base = dict(uid=self.uid, timeStamp=ts)
if flatten:
return base | self.head | self.data
return base | super().asDict()
def set(self, attr, value):
if attr in self.headFields:
if value is None:
value = ''
self.head[attr] = value
setattr(self, attr, value)
else:
raise AttributeError(attr)
@staticmethod
def prepareData(data, obj):
data.pop('uid', None)
data.pop('timeStamp', None)
return data
def update(self, data, overwrite=False):
if data is None:
return
if overwrite:
self.data = data
else:
self.data.update(data)
@property
def uid(self):
@ -51,6 +66,13 @@ class Record(shape.Record):
return ''
return str(self.trackId)
def __repr__(self):
return '<%s: %s>' % (self.__class__.__name__, self.asDict())
def asDict(self):
return dict(uid=self.uid, head=self.head, data=self.data,
timeStamp=str(self.timeStamp)[:19])
# shortcuts for compatibility with existing software:
def updateIndex(self, **kw):
@ -61,20 +83,15 @@ class Record(shape.Record):
def storeTrack(self):
self.container.update(self)
class Track(Record):
headFields: tuple[str, ...] = ('taskId', 'userName')
prefix = 'rec'
@registerContainerClass
class Container:
class Container(object):
itemFactory: type[Track] = Track
itemFactory = Track
tableName = 'tracks'
insertOnChange = True # always insert new track when data are changed
indexes: Indexes = None # is set in registerContainerClass()
#indexes = [('username',), ('taskid', 'username')] # explicit setting in subclass
indexes = None # default, will be overwritten by registerContainerClass()
#indexes = [('username',), ('taskid', 'username')] # or put explicitly in class
table = None
@ -120,24 +137,21 @@ class Container:
changed = False
if found.data != track.data:
found.update(track.data)
track.data = found.data
changed = True
if track.timeStamp is not None and found.timeStamp != track.timeStamp:
found.timeStamp = track.timeStamp
changed = True
if changed:
self.update(found, not track.timeStamp)
self.update(found)
track.trackId = found.trackId
track.timeStamp = found.timeStamp
return found.trackId
def insert(self, track, withTrackId=False):
t = self.table
values = self.setupValues(track, withTrackId)
stmt = t.insert().values(**values).returning(t.c.trackid, t.c.timestamp)
trackId, ts = self.session.execute(stmt).first()
stmt = t.insert().values(**values).returning(t.c.trackid)
trackId = self.session.execute(stmt).first()[0]
track.trackId = trackId
track.timeStamp = ts
self.storage.mark_changed()
return trackId

26
scopes/storage/user.py
View file

@ -1,26 +0,0 @@
# scopes.storage.user
from scopes.storage.common import registerContainerClass
from scopes.storage import concept
class User(concept.Concept):
headFields = ('name', 'login')
prefix = 'user'
@staticmethod
def prepareData(data, obj):
data = concept.Concept.prepareData(data, obj)
pw = data.pop('password', None)
if pw:
data['pw_hash'] = obj.hashPassword()
return data
@registerContainerClass
class Users(concept.Concepts):
itemFactory = User
indexes = [('name',), ('login',)]
tableName = 'users'

4
scopes/testing.py
View file

@ -1,4 +0,0 @@
# scopes.testing
"""Common stuff (config, fixtures, ...) for testing"""

48
scopes/tests/config.py
View file

@ -1,41 +1,29 @@
# py-scopes/tests/config.py
import logging
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_file='.env')
base_url: str = 'https://test.cy7.eu'
log_file: str = 'scopes/tests/log/scopes-test.log'
log_level: str = 'INFO'
log_format: str = '%(asctime)s %(levelname)s %(name)s %(message)s'
log_dateformat: str = '%Y-%m-%dT%H:%M:%S'
dbengine: str = 'postgresql+psycopg'
dbname: str = 'testdb'
dbuser: str = 'testuser'
dbpassword: str = 'secret'
dbschema: str = 'testing'
settings = Settings()
from os import getenv
import sys
base_url = settings.base_url
#from scopes.web.app import demo_app, zope_app
logging.basicConfig(filename=settings.log_file, level=settings.log_level,
format=settings.log_format, datefmt=settings.log_dateformat)
log_file = 'scopes/tests/log/scopes-test.log'
log_level = logging.INFO
log_format = '%(asctime)s %(levelname)s %(name)s %(message)s'
log_dateformat = '%Y-%m-%dT%H:%M:%S'
# SQLite storage settings
dbengine = settings.dbengine
dbname = settings.dbname
dbuser = settings.dbuser
dbpassword = settings.dbpassword
dbschema = settings.dbschema
# server / app settings
server_port = '8999'
base_url = 'testing:'
#app = zope_app
# storage settings
# SQLite
dbengine = 'sqlite'
dbname = 'var/test.db'
dbuser = None
dbpassword = None
dbschema = None
# special testing stuff
from scopes.tests import data_auth # add oidc URIs and keys to dummy_requests data

8
scopes/tests/test_postgres.py
View file

@ -6,7 +6,7 @@ import os, sys
sys.path = [os.path.dirname(__file__)] + sys.path
import unittest
from scopes.tests import tlib_storage, tlib_web
from scopes.tests import tlib_storage
from scopes.tests.util import setup_logging
from scopes.storage.db.postgres import StorageFactory
@ -38,12 +38,6 @@ class Test(unittest.TestCase):
def test_005_message(self):
tlib_storage.test_message(self, config)
def test_013_web(self):
tlib_web.test_app(self, config)
tlib_web.test_auth(self, config)
tlib_web.test_user_data(self, config)
def test_suite():
return unittest.TestSuite((
unittest.TestLoader().loadTestsFromTestCase(Test),

0
scopes/tests/xx_test_standard.py → scopes/tests/test_standard.py
View file

4
scopes/tests/tlib_storage.py
View file

@ -4,10 +4,6 @@
from datetime import datetime
from scopes.storage import concept, folder, message, topic, tracking
from scopes.util import keywords as k
# predefine keywords used as domain, action, class head values in messages
k('session system')
def test_tracking(self, config):

2
scopes/tests/tlib_web.py
View file

@ -42,7 +42,7 @@ def test_auth(self, config):
logger.info('test_auth keys: %s', keys)
def test_user_data(self, config):
from scopes.web.auth import user
from scopes.org import user
u1 = user.User(login='tst9', email='tst9@example.com',
firstName='Test', lastName='User')
xu1 = user.ExtUser(u1, idPrefix='test.')

5
scopes/tests/util.py
View file

@ -3,10 +3,9 @@
import logging
def setup_logging(config):
settings = config.settings
hdlr = logging.getLogger().handlers[-1]
logging.getLogger().removeHandler(hdlr) # remove NullHandler added by testrunner
logging.basicConfig(filename=settings.log_file, level=settings.log_level,
format=settings.log_format, datefmt=settings.log_dateformat)
logging.basicConfig(filename=config.log_file, level=config.log_level,
format=config.log_format, datefmt=config.log_dateformat)

43
scopes/util.py
View file

@ -1,53 +1,10 @@
# scopes.util
import base64
from collections.abc import Iterable
import hashlib
from secrets import choice
import string
# symbols and keywords
class Symbol(str):
def __repr__(self):
return f'<{self}>'
class Symbols:
def __init__(self, ns=None):
self.__prefix = ns + '.' if ns else ''
def __call__(self, *keys):
for segment in keys:
for key in segment.split():
if not hasattr(self, key):
setattr(self, key, Symbol(self.__prefix + key))
return self
def __getattr__(self, k):
raise AttributeError(k)
def __getitem__(self, k):
return getattr(self, k)
keywords = Symbols()
# list / iterable utilities
def flatten(*it, ignore_none=False):
accu = []
def doit(it):
if ignore_none and it is None:
return
if isinstance(it, Iterable):
for t in it:
doit(t)
else:
accu.append(it)
doit(it)
return accu
# random strings, hashes, encodings
# for authentication, encryption, and other purposes

24
tests/test.env
View file

@ -1,24 +0,0 @@
# tests/test.env
LOG_FILE=tests/log/scopes-test.log
BASE_URL=http://test
DBNAME=testdb
DBUSER=testuser
DBPASSWORD=secret
DBSCHEMA=testing
AUTH_METHOD=basic
# openssl rand -hex 32:
JWT_KEY=391a521ee5ba2b6ce67b8fdbb9f0ca88f40a87d3bc1e82974381b647bb0fc776
JWT_ALGORITHM=HS256
OIDC_PROVIDER=http://oidc.test
OIDC_CLIENT_ID=12345
OIDC_COOKIE_CRYPT=secret_crypt
OIDC_ORGANIZATION_ID=12346
OIDC_PROJECT_ID=12347

94
tests/test_api.py
View file

@ -1,94 +0,0 @@
# py-scopes: tests.test_api
"""Tests for the fastapi- (and asyncio-) based stuff"""
from scopes import config
from scopes.api.auth import config as auth_config
config.settings = config.Settings(_env_file='tests/test.env')
auth_config.settings = auth_config.Settings(_env_file='tests/test.env')
import pytest
from fastapi.testclient import TestClient
from httpx2 import ASGITransport, AsyncClient, BasicAuth
from scopes.api import main
from scopes.api.main import app
from scopes.storage import tracking, user as st_user
from scopes.api.auth import user as api_user
@pytest.fixture(scope='module')
def storage():
storage = config.StorageFactory(config.settings)(config.settings.dbschema)
storage.dropTable('tracks')
storage.dropTable('users')
storage.commit()
return storage
def async_client():
return AsyncClient(transport=ASGITransport(app=app), base_url='http://test')
#@pytest.mark.anyio
#async def test_root():
#async with async_client() as clt:
#response = await clt.get('/')
def test_root():
with TestClient(app) as clt:
response = clt.get('/')
assert response.status_code == 200
assert response.json() == {'Hello': 'World'}
def test_tracking(storage, caplog):
tracks = storage.create(tracking.Container)
tr01 = tracking.Track('t01', 'john')
tr01.update(dict(activity='testing'))
trid01 = tracks.save(tr01)
storage.commit()
with TestClient(app) as clt:
caplog.set_level(config.settings.log_level)
tracks = clt.get('/tracks').json()
assert len(tracks) == 1
tr = tracks[0]
assert tr['timeStamp'].startswith('202')
tr['timeStamp'] = '...'
assert tr == {'uid': 'rec-1', 'head': {'taskId': 't01', 'userName': 'john'},
'data': {'activity': 'testing'}, 'timeStamp': '...'}
resp = clt.post('/tracks', json={'head': {'taskId': 't02', 'userName': 'jim'},
'data': {'activity': 'implement post track'}})
assert resp.status_code == 200
tr02 = resp.json()
assert tr02['uid'] == 'rec-2'
#print('\n*** Logs: ', caplog.text)
#print('*** tr02', tr02)
def test_user(storage, caplog):
users = storage.create(st_user.Users)
pw = 'secret'
pw_hash = api_user.hashPassword(pw)
u01 = st_user.User('jim', 'jim', data=dict(display_name='Jim', pw_hash=pw_hash))
u01id = users.save(u01)
storage.commit()
with TestClient(app) as clt:
caplog.set_level(config.settings.log_level)
auth = BasicAuth(username='jim', password='secret')
users = clt.get('/users').json()
assert len(users) == 1
u01i = users[0]
assert u01i.get(pw_hash) is None
inp = {'name': 'jane', 'login': 'jane', 'display_name': 'Jane',
'email': 'jane@xmpl.org', 'password': 'secret'}
resp = clt.post('/users', json=inp, auth=auth)
assert resp.status_code == 200
u02 = resp.json()
assert u02['uid'] == 'user-2'
ts02 = u02.get('timeStamp')
assert ts02 is not None
del inp['password']
inp['display_name'] = 'Jane Doe'
resp = clt.post('/users', json=inp, auth=auth)
assert resp.status_code == 200
u02a = resp.json()
assert u02a['uid'] == 'user-2'
ts02a = u02a.get('timeStamp')
assert ts02a is not None

16
tests/test_config.py
View file

@ -1,16 +0,0 @@
# py-scopes: tests.test_config
"""Tests for scopes configuration settings via config.py and .env"""
import os
import pytest
from scopes import config
config.settings = config.Settings(_env_file='tests/test.env')
from scopes.config import settings
def test_config():
assert settings.dbname == 'testdb'
assert settings.dbschema == 'testing'

89
tests/test_core.py
View file

@ -1,89 +0,0 @@
# py-scopes: tests.test_core
""" Tests for scopes.core modules, including e.g. actor, message"""
import asyncio
import pytest
from scopes import core
from scopes.core import actor, environ
from scopes.core.message import Message
from scopes.util import keywords as k
k('data', 'hello', 'person', 'show', 'switch', 'quit', 'test')
@pytest.fixture
def collector():
return []
def show(msg, coll):
coll.append(msg)
def handle_basic(coll):
async def handle(msg):
match msg:
case k.show: show(msg, coll)
case k.switch: return handle_alt(coll)
case _: show(msg, coll)
return handle
def handle_alt(coll):
async def handle(msg):
match msg:
case k.show: show('switched', coll)
case _: await asyncio.sleep(0.01); show(msg, coll)
return handle
@pytest.mark.anyio
async def test_actor(collector):
mb1 = actor.create(handle_basic(collector))
mb2 = actor.create(handle_basic(collector))
msgs = [Message(k.test, k.hello), k.show, k.switch, k.show, "actor 1: Goodbye"]
await asyncio.gather(*[actor.send(mb1, m) for m in msgs])
await actor.send(mb2, "actor 2")
await actor.send(mb2, k.show)
await actor.send(mb2, k.quit)
await actor.send(mb2, k.quit)
await asyncio.sleep(0.01)
#print(collector)
assert len(collector) == 6
assert collector[-1] == 'actor 1: Goodbye'
def test_message():
msg = Message(k.test, k.data, 'person', '4711',
data=dict(firstName='Bob', lastName='Miller'))
assert msg.head['class'] is k.person
assert msg.addr() == (k.test, k.person, '4711')
assert msg.categ() == (k.test, k.person)
assert msg.asDict() == {'head': {'domain': k.test, 'action': k.data,
'class': k.person, 'item': '4711'},
'data': {'firstName': 'Bob', 'lastName': 'Miller'}}
assert msg.asDict(flatten=True) == {
'domain': k.test, 'action': k.data, 'class': k.person, 'item': '4711',
'firstName': 'Bob', 'lastName': 'Miller'}
@pytest.fixture
async def meta_env(collector):
async def collect(msg):
collector.append(msg)
return actor.create(collect)
@pytest.mark.anyio
async def test_core(meta_env, collector):
env = environ.create_env(meta_env)
zero = await core.create_zero(env)
await actor.send(env, Message(k.env, k.hello))
await actor.send(zero, Message(k.csys, k.hello))
await actor.send(zero, Message(k.csys, k.create,
data={k.connect: k.pred, k.address: (1, 1)}))
await asyncio.sleep(0.01)
assert len(collector) == 4
assert collector[0].action is k.created
assert collector[1].domain is k.env
assert collector[2].domain is k.csys
assert collector[3].action is k.created
#print(collector)

130
tests/test_storage.py
View file

@ -1,130 +0,0 @@
# py-scopes: tests.test_storage
""" Tests for the (PostgreSQL) storage-related functionalities"""
from scopes import config
config.settings = config.Settings(_env_file='tests/test.env')
from datetime import datetime
import pytest
from scopes.storage import concept, folder, message, topic, tracking
from scopes.util import keywords as k
# predefined keywords used as domain, action, class head values in messages
k('session system')
@pytest.fixture(scope='module')
def storage():
storage = config.StorageFactory(config.settings)(config.settings.dbschema)
storage.dropTable('tracks')
storage.dropTable('folders')
storage.dropTable('types')
storage.dropTable('preds')
storage.dropTable('rels')
storage.dropTable('topics')
storage.dropTable('messages')
storage.commit()
return storage
def test_tracking(storage):
tracks = storage.create(tracking.Container)
tr01 = tracking.Track('t01', 'john')
assert tr01.head == {'taskId': 't01', 'userName': 'john'}
assert tr01.taskId == 't01'
assert tr01.userName == 'john'
tr01.update(dict(activity='testing'))
trid01 = tracks.save(tr01)
assert trid01 > 0
tr01a = tracks[f'{trid01:07}']
assert tr01a.head == tr01.head
assert tr01a.trackId == trid01
assert tr01a.data.get('activity') == 'testing'
tr01a.update(dict(text='Set up unit tests.'))
tr01a.timeStamp = None
assert tracks.save(tr01a) > 0
tr01b = tracks.queryLast(taskId='t01')
assert tr01b.head == tr01.head
assert tr01b.trackId != trid01
assert tr01b.data.get('activity') == 'testing'
assert tr01b.data.get('text') == 'Set up unit tests.'
tr02 = tracking.Track('t02', 'jim', trackId=31, timeStamp=datetime(2023, 11, 30),
data=dict(activity='concept'))
trid02 = tracks.upsert(tr02)
assert trid02 == 31
assert tr02.uid == 'rec-31'
tr02.trackId = trid01
trid021 = tracks.upsert(tr02)
assert trid021 == trid01
assert tr02.uid == 'rec-' + str(trid01)
tr03 = storage.getItem('rec-31')
assert tr03.trackId == 31
n = tracks.remove(tr03)
assert n == 1
assert tracks.get(31) == None
storage.commit()
def test_folder(storage):
root = folder.Root(storage)
assert list(root.keys()) == []
root['top'] = folder.Folder()
assert list(root.keys()) == ['top']
top = root['top']
top['child1'] = folder.Folder(data=dict(title='First Child'))
assert list(top.keys()) == ['child1']
ch1 = top['child1']
assert ch1.parent == top.rid
assert list(top.keys()) == ['child1']
ch1.set('name', 'level2-item1')
ch1.storeTrack()
storage.commit()
def test_type(storage):
concept.setupCoreTypes(storage)
types = storage.getContainer(concept.Type)
tps = list(types.query())
assert len(tps) == 8
tfolder = types.queryLast(name='folder')
fldrs = list(tfolder.values())
assert len(fldrs) == 2
assert fldrs[0].name == 'top'
storage.commit()
def test_topic(storage):
rels = storage.getContainer(concept.Triple)
topics = storage.getContainer(topic.Topic)
types = storage.getContainer(concept.Type)
concept.storePredicate(storage, concept.defaultPredicate)
root = folder.Root(storage)
root['top']['topics'] = ftopics = folder.Folder()
ttopic = types.queryLast(name='topic')
assert ttopic.name == 'topic'
ftopics.setTarget(ttopic)
assert ftopics.getTarget().name == 'topic'
tp_itc = topic.Topic('itc', data=dict(
title='ITC', description='Information and Communication Technology'))
topics.save(tp_itc)
tp_proglang = topic.Topic('prog_lang', data=dict(
title='Programming Languages',
description='Programming Languages'))
topics.save(tp_proglang)
tp_itc.addChild(tp_proglang)
c = list(tp_itc.children())
assert c[0].name == 'prog_lang'
storage.commit()
def test_message(storage):
messages = storage.create(message.Messages)
m01 = message.Message('system', 'data', 'session', 'V1_317784226621611853')
m01.update(dict(userid='tst9'))
mid01 = messages.save(m01)
msgs = list(messages.query())
assert msgs[0].domain is k.system
assert len(msgs) == 1
storage.commit()

30
tests/test_util.py
View file

@ -1,30 +0,0 @@
# py-scopes: tests.test_util
"""Tests for the scopes.util package"""
import pytest
from scopes.util import keywords as k, flatten, Symbols
k('dummy other', 'cd', 'new', 'erp')
dom = Symbols('dom')('crm erp sys')
def test_keywords():
assert k.dummy == 'dummy'
assert k.dummy is not 'dummy'
assert k.dummy is k.dummy
assert k.dummy is not k.other
assert getattr(k, "abcd"[-2:]) is k.cd
assert k['dummy'] is k.dummy
assert k['new'] is k['new']
assert dom.erp == 'dom.erp'
assert dom.erp is not 'dom.erp'
assert repr(dom.erp) == '<dom.erp>'
assert dom.erp != k.erp
def test_flatten():
lst = (1, (None, 2), (3, (4, 5)))
assert flatten(lst) == [1, None, 2, 3, 4, 5]
assert flatten(42, lst, (6, 7), ignore_none=True) == [42, 1, 2, 3, 4, 5, 6, 7]