So I need myresource in every op. How do I do this without having to specify it in every op, but in the job definition instead? I think this is possible, but I cannot find a specification in the documentation for exactly what the job config looks like. Thanks!
#op()
def op1(context):
# This throws an error
res = context.resources.myresource
return
#op()
def op2(context):
res = context.resources.myresource
return
#op()
def op3(context):
res = context.resources.myresource
return
#op()
def op4(context):
res = context.resources.myresource
return
#job(
resource_defs = {"myresource" : myresource.configured({"arg" : "whatever"})}
)
def myjob():
op1()
op2()
op3()
op4()
Related
Suppose I have a dag as follows:
def my_dag():
#task
def task_a():
# Some code here
return
#task
def task_b():
# Some code here
return
task_a()
task_b()
my_dag_instance = my_dag()
I would like to set task_b downstream from task_a. That is, I want task_b to execute only if task_a has executed successfully. How can I do so? Note that task_a does not return anything.
I tried task_a >> task_b right before the last line of my code (my_dag_instance = my_dag()) yet got an error along the lines of " TaskDecorator >> TaskDecorator is invalid operation". Any advice?
Indeed, you can do that, inside the dag context, and with task instance instead of task method reference:
def my_dag():
#task
def task_a():
# Some code here
return
#task
def task_b():
# Some code here
return
task_a() >> task_b()
# or
# A = task_a()
# B = task_b()
# A >> B
my_dag_instance = my_dag()
Recently, I run the code released by other authors. They utilized chainer v1.3, but I installed v4. When I run the code, it errors that Attribute Errors: MomentSGD optimizer has no attribute prepare. Here I post the codes of this part:
class BaseModel(chainer.Chain):
loss = None
accuracy = None
gpu_mode = False
_train = False
def __call__(self, *arg_list, **arg_dict):
raise NotImplementedError()
def clear(self):
self.loss = None
self.accuracy = None
def train(self, data, optimizer):
self._train = True
optimizer.update(self, data)
if self.accuracy is None:
return float(self.loss.data)
else:
return float(self.loss.data), float(self.accuracy.data)
def validate(self, data):
self._train = False
self(data)
if self.accuracy is None:
return float(self.loss.data)
else:
return float(self.loss.data), float(self.accuracy.data)
def test(self, data):
self._train = False
raise NotImplementedError()
def save(self, fname):
serializers.save_hdf5(fname, self)
def load(self, fname):
serializers.load_hdf5(fname, self)
def cache(self):
self.to_cpu()
cached_model = self.copy()
self.to_gpu()
return cached_model
# this part is the error part
def setup(self, optimizer):
self.to_gpu()
optimizer.target = self
optimizer.prepare()
def to_cpu(self):
if not self.gpu_mode:
return
super(BaseModel, self).to_cpu()
self.gpu_mode = False
def to_gpu(self):
if self.gpu_mode:
return
super(BaseModel, self).to_gpu()
self.gpu_mode = True
Newer version of chainer uses setup method to initialize optimizer.
Can you try modifing your code as follows?
def setup(self, optimizer):
self.to_gpu()
optimizer.setup(self)
On a simple grammar I am in the bad situation that one of my ParseActions is not called.
For me this is strange as parseActions of a base symbol ("logic_oper") and a derived symbol ("cmd_line") are called correctly. Just "pa_logic_cmd" is not called. You can see this on the output which is included at the end of the code.
As there is no exception on parsing the input string, I am assuming that the grammar is (basically) correct.
import io, sys
import pyparsing as pp
def diag(msg, t):
print("%s: %s" % (msg , str(t)) )
def pa_logic_oper(t): diag('logic_oper', t)
def pa_operand(t): diag('operand', t)
def pa_ident(t): diag('ident', t)
def pa_logic_cmd(t): diag('>>>>>> logic_cmd', t)
def pa_cmd_line(t): diag('cmd_line', t)
def make_grammar():
semi = pp.Literal(';')
ident = pp.Word(pp.alphas, pp.alphanums).setParseAction(pa_ident)
operand = (ident).setParseAction(pa_operand)
op_and = pp.Keyword('A')
op_or = pp.Keyword('O')
logic_oper = (( op_and | op_or) + pp.Optional(operand))
logic_oper.setParseAction(pa_logic_oper)
logic_cmd = logic_oper + pp.Suppress(semi)
logic_cmd.setParseAction(pa_logic_cmd)
cmd_line = (logic_cmd)
cmd_line.setParseAction(pa_cmd_line)
grammar = pp.OneOrMore(cmd_line) + pp.StringEnd()
return grammar
if __name__ == "__main__":
inp_str = '''
A param1;
O param2;
A ;
'''
grammar = make_grammar()
print( "pp-version:" + pp.__version__)
parse_res = grammar.parseString( inp_str )
'''USAGE/Output: python test_4.py
pp-version:2.0.3
operand: ['param1']
logic_oper: ['A', 'param1']
cmd_line: ['A', 'param1']
operand: ['param2']
logic_oper: ['O', 'param2']
cmd_line: ['O', 'param2']
logic_oper: ['A']
cmd_line: ['A']
'''
Can anybody give me a hint on this parseAction problem?
Thanks,
The problem is here:
cmd_line = (logic_cmd)
cmd_line.setParseAction(pa_cmd_line)
The first line assigns cmd_line to be the same expression as logic_cmd. You can verify by adding this line:
print("???", cmd_line is logic_cmd)
Then the second line calls setParseAction, which overwrites the parse action of logic_cmd, so the pa_logic_cmd will never get called.
Remove the second line, since you are already testing the calling of the parse action with pa_logic_cmd. You could change to using the addParseAction method instead, but to my mind that is an invalid test (adding 2 parse actions to the same pyparsing expression object).
Or, change the definition of cmd_line to:
cmd_line = pp.Group(logic_cmd)
Now you will have wrapped logic_cmd inside another expression, and you can then independently set and test the running of parse actions on the two different expressions.
Let's say I have a closure:
def increment = {value, step ->
value + step
}
Now I want to loop over every item of my integers collection, increment it with 5, and save new elements to a new collection:
def numbers = [1..10]
def biggerNumbers = numbers.collect {
it + 5
}
And now I want to achieve the same result but by means of using increment closure. How can I do this?
Should be something like this (wrong code below):
def biggerNumbers = numbers.collect increment(it, 5) //what's the correct name of 'it'??
The solution to your problem would be nesting your call of increment in a closure:
def biggerNumbers = numbers.collect {increment(it, 5)}
If you wanted to pass a premade closure to the collect you should have made it compatible with collect - accepting a single parameter that is:
def incrementByFive = {it + 5}
def biggerNumbers = numbers.collect incrementByFive
mojojojo has the right answer, but just thought I'd add that this looks like a good candidate for currying (specifically using rcurry)
If you have:
def increment = {value, step ->
value + step
}
You can then curry the right-hand parameter of this function with:
def incrementByFive = increment.rcurry 5
And then, you can do:
def numbers = 1..10
def biggerNumbers = numbers.collect incrementByFive
Just thought it might be of interest ;-)
The main issue is that [1..10] creates a List<IntRange> which you are trying to increment. You should collect on the IntRange directly (note the lack of brackets):
(1..10).collect { it + 5 }
Or with curry:
def sum = { a, b -> a + b }
(1..10).collect(sum.curry(5))
I find functools.partial to be extremely useful, but I would like to be able to freeze arguments out of order (the argument you want to freeze is not always the first one) and I'd like to be able to apply it to several methods on a class at once, to make a proxy object that has the same methods as the underlying object except with some of its methods parameters being frozen (think of it as generalizing partial to apply to classes). And I'd prefer to do this without editing the original object, just like partial doesn't change its original function.
I've managed to scrap together a version of functools.partial called 'bind' that lets me specify parameters out of order by passing them by keyword argument. That part works:
>>> def foo(x, y):
... print x, y
...
>>> bar = bind(foo, y=3)
>>> bar(2)
2 3
But my proxy class does not work, and I'm not sure why:
>>> class Foo(object):
... def bar(self, x, y):
... print x, y
...
>>> a = Foo()
>>> b = PureProxy(a, bar=bind(Foo.bar, y=3))
>>> b.bar(2)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
TypeError: bar() takes exactly 3 arguments (2 given)
I'm probably doing this all sorts of wrong because I'm just going by what I've pieced together from random documentation, blogs, and running dir() on all the pieces. Suggestions both on how to make this work and better ways to implement it would be appreciated ;) One detail I'm unsure about is how this should all interact with descriptors. Code follows.
from types import MethodType
class PureProxy(object):
def __init__(self, underlying, **substitutions):
self.underlying = underlying
for name in substitutions:
subst_attr = substitutions[name]
if hasattr(subst_attr, "underlying"):
setattr(self, name, MethodType(subst_attr, self, PureProxy))
def __getattribute__(self, name):
return getattr(object.__getattribute__(self, "underlying"), name)
def bind(f, *args, **kwargs):
""" Lets you freeze arguments of a function be certain values. Unlike
functools.partial, you can freeze arguments by name, which has the bonus
of letting you freeze them out of order. args will be treated just like
partial, but kwargs will properly take into account if you are specifying
a regular argument by name. """
argspec = inspect.getargspec(f)
argdict = copy(kwargs)
if hasattr(f, "im_func"):
f = f.im_func
args_idx = 0
for arg in argspec.args:
if args_idx >= len(args):
break
argdict[arg] = args[args_idx]
args_idx += 1
num_plugged = args_idx
def new_func(*inner_args, **inner_kwargs):
args_idx = 0
for arg in argspec.args[num_plugged:]:
if arg in argdict:
continue
if args_idx >= len(inner_args):
# We can't raise an error here because some remaining arguments
# may have been passed in by keyword.
break
argdict[arg] = inner_args[args_idx]
args_idx += 1
f(**dict(argdict, **inner_kwargs))
new_func.underlying = f
return new_func
Update: In case anyone can benefit, here's the final implementation I went with:
from types import MethodType
class PureProxy(object):
""" Intended usage:
>>> class Foo(object):
... def bar(self, x, y):
... print x, y
...
>>> a = Foo()
>>> b = PureProxy(a, bar=FreezeArgs(y=3))
>>> b.bar(1)
1 3
"""
def __init__(self, underlying, **substitutions):
self.underlying = underlying
for name in substitutions:
subst_attr = substitutions[name]
if isinstance(subst_attr, FreezeArgs):
underlying_func = getattr(underlying, name)
new_method_func = bind(underlying_func, *subst_attr.args, **subst_attr.kwargs)
setattr(self, name, MethodType(new_method_func, self, PureProxy))
def __getattr__(self, name):
return getattr(self.underlying, name)
class FreezeArgs(object):
def __init__(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
def bind(f, *args, **kwargs):
""" Lets you freeze arguments of a function be certain values. Unlike
functools.partial, you can freeze arguments by name, which has the bonus
of letting you freeze them out of order. args will be treated just like
partial, but kwargs will properly take into account if you are specifying
a regular argument by name. """
argspec = inspect.getargspec(f)
argdict = copy(kwargs)
if hasattr(f, "im_func"):
f = f.im_func
args_idx = 0
for arg in argspec.args:
if args_idx >= len(args):
break
argdict[arg] = args[args_idx]
args_idx += 1
num_plugged = args_idx
def new_func(*inner_args, **inner_kwargs):
args_idx = 0
for arg in argspec.args[num_plugged:]:
if arg in argdict:
continue
if args_idx >= len(inner_args):
# We can't raise an error here because some remaining arguments
# may have been passed in by keyword.
break
argdict[arg] = inner_args[args_idx]
args_idx += 1
f(**dict(argdict, **inner_kwargs))
return new_func
You're "binding too deep": change def __getattribute__(self, name): to def __getattr__(self, name): in class PureProxy. __getattribute__ intercepts every attribute access and so bypasses everything that you've set with setattr(self, name, ... making those setattr bereft of any effect, which obviously's not what you want; __getattr__ is called only for access to attributes not otherwise defined so those setattr calls become "operative" & useful.
In the body of that override, you can and should also change object.__getattribute__(self, "underlying") to self.underlying (since you're not overriding __getattribute__ any more). There are other changes I'd suggest (enumerate in lieu of the low-level logic you're using for counters, etc) but they wouldn't change the semantics.
With the change I suggest, your sample code works (you'll have to keep testing with more subtle cases of course). BTW, the way I debugged this was simply to stick in print statements in the appropriate places (a jurassic=era approach but still my favorite;-).