I am writing a simplistic ORM that would require me to override the methods inside the child classes. Depending on the context, I am either expecting to access the model through the class methods, or through the instance methods, therefore I can not simply override them. I believe this code describes the question well enough:
class A:
#classmethod
def key_fn(cls, id):
raise NotImplementedError('')
#classmethod
def load_all(cls):
yield from db_fetch_prefix(cls.key_fn('')):
class B(A):
#classmethod
def key_fn(cls, id):
return f'/keys/{id}'
# how do I make sure B.key_fn is called here?
B.load_all()
Your B.key_fn would be called indeed. But your load_all returns a generator, because you let it yield from db_fetch_prefix. You can check this by running print(B.load_all()) at the end. The output will be:
python .\clsss.py
<generator object A.load_all at 0x0521A330>
I don't know what you want to achieve by using yield from. But an example that shows overriding of classmethods in subclasses is possible would be:
class A:
#classmethod
def key_fn(cls, id):
raise NotImplementedError('')
#classmethod
def load_all(cls):
return cls.key_fn('foo')
class B(A):
#classmethod
def key_fn(cls, id):
return f'/keys/{id}'
print(B.load_all()) # prints "/keys/foo"
Guessing wildly that you want to apply the key_fn to each item yielded by your generator db_fetch_prefix (abbreviated to fetch below), the code below shows B.key_fn will be used in the presence of generators as well.
def fetch(callback):
for i in range(5): # fake DB fetch
yield callback(i)
class A:
#classmethod
def key_fn(cls, id):
raise NotImplementedError('')
#classmethod
def load_all(cls):
yield from fetch(cls.key_fn)
class B(A):
#classmethod
def key_fn(cls, id):
return f'/keys/{id}'
print(list(B.load_all())) # prints ['/keys/0', '/keys/1', '/keys/2', '/keys/3', '/keys/4']
Related
Is there a way to convert a pydantic model to query parameters in fastapi?
Some of my endpoints pass parameters via the body, but some others pass them directly in the query. All this endpoints share the same data model, for example:
class Model(BaseModel):
x: str
y: str
I would like to avoid duplicating my definition of this model in the definition of my "query-parameters endpoints", like for example test_query in this code:
class Model(BaseModel):
x: str
y: str
#app.post("/test-body")
def test_body(model: Model): pass
#app.post("/test-query-params")
def test_query(x: str, y: str): pass
What's the cleanest way of doing this?
The documentation gives a shortcut to avoid this kind of repetitions. In this case, it would give:
from fastapi import Depends
#app.post("/test-query-params")
def test_query(model: Model = Depends()): pass
This will allow you to request /test-query-params?x=1&y=2 and will also produce the correct OpenAPI description for this endpoint.
Similar solutions can be used for using Pydantic models as form-data descriptors.
Special case that isn't mentioned in the documentation for Query Parameters Lists, for example with:
/members?member_ids=1&member_ids=2
The answer provided by #cglacet will unfortunately ignore the array for such a model:
class Model(BaseModel):
member_ids: List[str]
You need to modify your model like so:
class Model(BaseModel):
member_ids: List[str] = Field(Query([]))
Answer from #fnep on GitHub here
This solution is very apt if your schema is "minimal".
But, when it comes to a complicated one like this, Set description for query parameter in swagger doc using Pydantic model, it is better to use a "custom dependency class"
from fastapi import Depends, FastAPI, Query
app = FastAPI()
class Model:
def __init__(
self,
y: str,
x: str = Query(
default='default for X',
title='Title for X',
deprecated=True
)
):
self.x = x
self.y = y
#app.post("/test-body")
def test_body(model: Model = Depends()):
return model
If you are using this method, you will have more control over the OpenAPI doc.
#cglacet 's answer is simple and works, but it will raise pydantic ValidationError when validation fail and not gonna pass the error to client.
You can find reason here.
This works and pass message to client. Code from here.
import inspect
from fastapi import Query, FastAPI, Depends
from pydantic import BaseModel, ValidationError
from fastapi.exceptions import RequestValidationError
class QueryBaseModel(BaseModel):
def __init_subclass__(cls, *args, **kwargs):
field_default = Query(...)
new_params = []
for field in cls.__fields__.values():
default = Query(field.default) if not field.required else field_default
annotation = inspect.Parameter.empty
new_params.append(
inspect.Parameter(
field.alias,
inspect.Parameter.POSITIONAL_ONLY,
default=default,
annotation=annotation,
)
)
async def _as_query(**data):
try:
return cls(**data)
except ValidationError as e:
raise RequestValidationError(e.raw_errors)
sig = inspect.signature(_as_query)
sig = sig.replace(parameters=new_params)
_as_query.__signature__ = sig # type: ignore
setattr(cls, "as_query", _as_query)
#staticmethod
def as_query(parameters: list) -> "QueryBaseModel":
raise NotImplementedError
class ParamModel(QueryBaseModel):
start_datetime: datetime
app = FastAPI()
#app.get("/api")
def test(q_param: ParamModel: Depends(ParamModel.as_query))
start_datetime = q_param.start_datetime
...
return {}
Is there a way to print all of a function's variables? I want to build a custom debugging decorator, and can't seem to find what I'm looking for. I'm assuming there is some dunder method for this? So for a function:
def debugger(func):
#functools.wraps(func)
def wrapper(*args, **kwargs):
print(func.__funcVariables__) #Some dunder method that Prints all variables contained in func
return func
return wrapper
#debugger
def my_func():
x = 'foo'
y = 'bar'
I would want 'foo' and 'bar' printed to the console from the decorator. How can I achieve this?
It sounds like you're looking for function.__code__.co_varnames, which is a tuple of the names of the functions arguments and local variables. This is documented with the rest of the code introspection tools in the documentation for the inspect module
def debugger(func):
#functools.wraps(func)
def wrapper(*args, **kwargs):
print(func.__code__.co_varnames)
return func
return wrapper
#debugger
def my_func():
x = 'foo'
y = 'bar'
I am trying to fix some methods annotations on magic and normal methods. For example, I have some cases like:
```
class Headers(typing.Mapping[str, str]):
...
def __contains__(self, key: str) -> bool:
...
return False
def keys(self) -> typing.List[str]:
...
return ['a', 'b']
```
and when I run mypy somefile.py --disallow-untyped-defs I have the following errors:
error: Argument 1 of "__contains__" incompatible with supertype "Mapping"
error: Argument 1 of "__contains__" incompatible with supertype "Container"
error: Return type of "keys" incompatible with supertype "Mapping"
What I understand is that I need to override the methods using the #override decorator and I need to respect the order of inheritance. Is it correct?
If my assumption is correct, Is there any place in which I can find the exact signatures of the parent classes?
After asking the question on mypy, the answer was:
Subclassing typing.Mapping[str, str], I'd assume that the function
signature for the argument key in contains ought to match the
generic type?
contains isn't a generic method -- it's defined to have the type signature contains(self, key: object) -> bool. You can check this on typeshed. The reason why contains is defined this way is because doing things like 1 in {"foo": "bar"} is technically legal.
Subclassing def contains(self, key) to def contains(self, key:
str) is in any case more specific. A more specific subtype doesn't
violate Liskov, no?
When you're overriding a function, it's ok to make the argument types more general and the return types more specific. That is, the argument types should be contravariant and the return types covariant.
If we did not follow the rule, we could end up introducing bugs in our code. For example:
class Parent:
def foo(self, x: object) -> None: ...
class Child(Parent):
def foo(self, x: str) -> None: ...
def test(x: Parent) -> None:
x.foo(300) # Safe if 'x' is actually a Parent, not safe if `x` is actually a Child.
test(Child())
Because we broke liskov, passing in an instance of Child into test ended up introducing a bug.
Basically if I use Any for key on __contains__ method is correct and mypy won't complaint :
def __contains__(self, key: typing.Any) -> bool:
...
return False
You can follow the conversation here
Imagine having a function, which handles a heavy computational job, that we wish to execute asynchronously in a Tornado application context. Moreover, we would like to lazily evaluate the function, by storing its results to the disk, and not rerunning the function twice for the same arguments.
Without caching the result (memoization) one would do the following:
def complex_computation(arguments):
...
return result
#gen.coroutine
def complex_computation_caller(arguments):
...
result = complex_computation(arguments)
raise gen.Return(result)
Assume to achieve function memoization, we choose Memory class from joblib. By simply decorating the function with #mem.cache the function can easily be memoized:
#mem.cache
def complex_computation(arguments):
...
return result
where mem can be something like mem = Memory(cachedir=get_cache_dir()).
Now consider combining the two, where we execute the computationally complex function on an executor:
class TaskRunner(object):
def __init__(self, loop=None, number_of_workers=1):
self.executor = futures.ThreadPoolExecutor(number_of_workers)
self.loop = loop or IOLoop.instance()
#run_on_executor
def run(self, func, *args, **kwargs):
return func(*args, **kwargs)
mem = Memory(cachedir=get_cache_dir())
_runner = TaskRunner(1)
#mem.cache
def complex_computation(arguments):
...
return result
#gen.coroutine
def complex_computation_caller(arguments):
result = yield _runner.run(complex_computation, arguments)
...
raise gen.Return(result)
So the first question is whether the aforementioned approach is technically correct?
Now let's consider the following scenario:
#gen.coroutine
def first_coroutine(arguments):
...
result = yield second_coroutine(arguments)
raise gen.Return(result)
#gen.coroutine
def second_coroutine(arguments):
...
result = yield third_coroutine(arguments)
raise gen.Return(result)
The second question is how one can memoize second_coroutine? Is it correct to do something like:
#gen.coroutine
def first_coroutine(arguments):
...
mem = Memory(cachedir=get_cache_dir())
mem_second_coroutine = mem(second_coroutine)
result = yield mem_second_coroutine(arguments)
raise gen.Return(result)
#gen.coroutine
def second_coroutine(arguments):
...
result = yield third_coroutine(arguments)
raise gen.Return(result)
[UPDATE I] Caching and reusing a function result in Tornado discusses using functools.lru_cache or repoze.lru.lru_cache as a solution for second question.
The Future objects returned by Tornado coroutines are reusable, so it generally works to use in-memory caches such as functools.lru_cache, as explained in this question. Just be sure to put the caching decorator before #gen.coroutine.
On-disk caching (which seems to be implied by the cachedir argument to Memory) is trickier, since Future objects cannot generally be written to disk. Your TaskRunner example should work, but it's doing something fundamentally different from the others because complex_calculation is not a coroutine. Your last example will not work, because it's trying to put the Future object in the cache.
Instead, if you want to cache things with a decorator, you'll need a decorator that wraps the inner coroutine with a second coroutine. Something like this:
def cached_coroutine(f):
#gen.coroutine
def wrapped(*args):
if args in cache:
return cache[args]
result = yield f(*args)
cache[args] = f
return result
return wrapped
I'm using pytest.mark to give my tests kwargs. However, if I use the same mark on both the class and a test within the class, the class's mark overrides the mark on the function when the same kwargs are used for both.
import pytest
animal = pytest.mark.animal
#animal(species='croc') # Mark the class with a kwarg
class TestClass(object):
#animal(species='hippo') # Mark the function with new kwarg
def test_function(self):
pass
#pytest.fixture(autouse=True) # Use a fixture to inspect my function
def animal_inspector(request):
print request.function.animal.kwargs # Show how the function object got marked
# prints {'species': 'croc'} but the function was marked with 'hippo'
Where'd my hippo go and how can I get him back?
There are unfortunately various pytest bugs related to this, I'm guessing you're running into one of them. The ones I found are related to subclassing which you don't do there though.
So I've been digging around in the pytest code and figured out why this is happening. The marks on the functions are applied to the function at import time but the class and module level marks don't get applied on the function level until test collection. Function marks happen first and add their kwargs to the function. Then class marks overwrite any same kwargs and module marks further overwrite any matching kwargs.
My solution was to simply create my own modified MarkDecorator that filters kwargs before they are added to the marks. Basically, whatever kwarg values get set first (which seems to always be by a function decorator) will always be the value on the mark. Ideally I think this functionality should be added in the MarkInfo class but since my code wasn't creating instances of that I went with what I was creating instances of: MarkDecorator. Note that I only change two lines from the source code (the bits about keys_to_add).
from _pytest.mark import istestfunc, MarkInfo
import inspect
class TestMarker(object): # Modified MarkDecorator class
def __init__(self, name, args=None, kwargs=None):
self.name = name
self.args = args or ()
self.kwargs = kwargs or {}
#property
def markname(self):
return self.name # for backward-compat (2.4.1 had this attr)
def __repr__(self):
d = self.__dict__.copy()
name = d.pop('name')
return "<MarkDecorator %r %r>" % (name, d)
def __call__(self, *args, **kwargs):
""" if passed a single callable argument: decorate it with mark info.
otherwise add *args/**kwargs in-place to mark information. """
if args and not kwargs:
func = args[0]
is_class = inspect.isclass(func)
if len(args) == 1 and (istestfunc(func) or is_class):
if is_class:
if hasattr(func, 'pytestmark'):
mark_list = func.pytestmark
if not isinstance(mark_list, list):
mark_list = [mark_list]
mark_list = mark_list + [self]
func.pytestmark = mark_list
else:
func.pytestmark = [self]
else:
holder = getattr(func, self.name, None)
if holder is None:
holder = MarkInfo(
self.name, self.args, self.kwargs
)
setattr(func, self.name, holder)
else:
# Don't set kwargs that already exist on the mark
keys_to_add = {key: value for key, value in self.kwargs.items() if key not in holder.kwargs}
holder.add(self.args, keys_to_add)
return func
kw = self.kwargs.copy()
kw.update(kwargs)
args = self.args + args
return self.__class__(self.name, args=args, kwargs=kw)
# Create my Mark instance. Note my modified mark class must be imported to be used
animal = TestMarker(name='animal')
# Apply it to class and function
#animal(species='croc') # Mark the class with a kwarg
class TestClass(object):
#animal(species='hippo') # Mark the function with new kwarg
def test_function(self):
pass
# Now prints {'species': 'hippo'} Yay!