"maskrcnn_benchmark"s github
Here is the source code for "FrozenBatchNorm2d"
import torch
from torch import nn
class FrozenBatchNorm2d(nn.Module):
def __init__(self, n):
super(FrozenBatchNorm2d, self).__init__()
self.register_buffer("weight", torch.ones(n))
self.register_buffer("bias", torch.zeros(n))
self.register_buffer("running_mean", torch.zeros(n))
self.register_buffer("running_var", torch.ones(n))
def forward(self, x):
scale = self.weight * self.running_var.rsqrt()
bias = self.bias - self.running_mean * scale
scale = scale.reshape(1, -1, 1, 1)
bias = bias.reshape(1, -1, 1, 1)
return x * scale + bias
When I put this function in my script, I found that this function had almost no effect.
Here is my usage
import torch.nn as nn
import torch
class FrozenBatchNorm2d(nn.Module):
"""
BatchNorm2d where the batch statistics and the affine parameters
are fixed
"""
def __init__(self, n):
super(FrozenBatchNorm2d, self).__init__()
self.register_buffer("weight", torch.ones(n))
self.register_buffer("bias", torch.zeros(n))
self.register_buffer("running_mean", torch.zeros(n))
self.register_buffer("running_var", torch.ones(n))
def forward(self, x):
scale = self.weight * self.running_var.rsqrt()
bias = self.bias - self.running_mean * scale
scale = scale.reshape(1, -1, 1, 1)
bias = bias.reshape(1, -1, 1, 1)
print(scale.shape,bias.shape)
return x * scale + bias
a=FrozenBatchNorm2d((1,2))
a(torch.tensor([1,2,3]))
The running result is different from what I thought.
So can someone tell me what this function exactly does?
I will appreciate it if someone could help me.
"register_buffer" means open an RAM for some parameters which couldn't be optimized or changed during the tranning process, in another word, the "weight","bias","running_mean","running_var" are consistent values. Hence, that is the reason why this rebuild batchnorm method could be called FrozenBatchnorm2d. It's my explan, hope it can help you.
Related
I have a blackbox solver which is wrapped as explicit component and the objective function and constraints are calculated in the blackbox solver and output. These are taken to a constraint components that has an equality constraint defined such that at any iteration, these constraints are satisifed. I am using finite difference to approximate the partial derivatives. However, I get this SLSQP error "Positive directional derivative for linesearch". From S.O., I understand that this error translates - optimizer could not find a direction to move to and also couldn't verify if the results are minimum. I found that for some iterations derivative is 'None' and it was 'None' at least a few times before it threw this error. Is it because the constraints are calculated in the black box solver? or is it because 'fd' for approximation is not working for non linear constraints? or both? A problem summary is attached for reference.
from PowerHarvest import *
from HydroDynamics import *
from SatelliteComms import *
from Propulsion import *
from Constraints import *
from SystemCost import *
class MDA(Group):
"""Multidisciplinary Analysis Group"""
def __init__(self, derivative_method='fd', **kwargs):
super(MDA, self).__init__(**kwargs)
self.derivative_method = 'fd'
def setup(self):
cycle = self.add_subsystem('cycle',Group(), promotes = ["*"])
cycle.nonlinear_solver = om.NewtonSolver(solve_subsystems = True)
cycle.nonlinear_solver.options['atol'] = 1e-6
cycle.add_subsystem('Hydro', Hydro(),promotes = ["*"]) #This is a blackbox explicit component!
cycle.add_subsystem('Propulsion_system', Propulsion(),promotes = ["*"])
cycle.add_subsystem('PowerHarvest_system',PowerHarvest(),promotes = ["*"])
cycle.add_subsystem("SatelitteComs_system", SatelitteComs(),promotes = ["*"])
cycle.nonlinear_solver.options['atol'] = 1.0e-5
cycle.nonlinear_solver.options['maxiter'] = 500
cycle.nonlinear_solver.options['iprint'] = 2
#Add constraint on the each subsytem if possible
#cycle.add_constraint('',om.ex)
self.add_subsystem('PowerConstraints_system', PowerConstraints(), promotes=["*"])
self.add_subsystem('BodyConstraints_system', BodyConstraint(),promotes = ["*"])
self.add_subsystem('SystemCost_system',SystemCost(), promotes = ['*'])
self.add_constraint('A_PV', upper = 100, units = 'm**2')
#these constraints are output of the blackbox solver!
self.add_constraint('AreaCon', upper = 0)
self.add_constraint('massCon',equals = 0)
self.add_constraint('P_Load', upper = 0) # Solar generates just enough for everything no storing!
self.add_constraint('DraughtCon', lower = 0.5 )
self.add_constraint('GMCon', lower = 0.01) #should be positive
#self.add_constraint('theta', upper = 0.14, lower = 0.1)
self.add_constraint('Amplitude_Con',upper = -0.1) #amplitude differenc
Added. Run script
import openmdao.api as om
from geom_utils import *
from openmdao.api import Problem, Group, ExplicitComponent,ImplicitComponent, IndepVarComp, ExecComp,\
NonlinearBlockGS, ScipyOptimizeDriver,NewtonSolver,DirectSolver,ScipyKrylov
import os
import numpy as np
from types import FunctionType
from geom_utils import *
from capytaine.meshes.meshes import Mesh
from pprint import pprint
from PowerHarvest import *
from HydroDynamics import *
from SatelliteComms import *
from Propulsion import *
from Constraints import *
from SystemCost import *
from PEARLMDA import *
if __name__ == '__main__':
prob = Problem()
model = prob.model = MDA()
prob.driver = ScipyOptimizeDriver(optimizer = 'SLSQP')
# prob.model.nonlinear_solver = om.NonlinearBlockGS()
#prob.driver.options['optimizer'] = 'COBYLA'
prob.driver.options['tol'] = 1e-5
prob.model.add_design_var('Df', lower= 6.0, upper=20.0, units = "m")
prob.model.add_design_var('tf', lower=1.0, upper=4.0, units = "m")
#prob.model.add_design_var('submergence', upper = -0.9)
prob.model.add_design_var('Vs', lower=1, upper=2, units = "m/s") #make sure the lower, upper are according to their units.
prob.model.add_design_var('ld', lower = 3, upper = 7, units = 'm' )
prob.model.add_objective('cost_per_byte' )
newton = om.NewtonSolver(solve_subsystems=True)
newton.linesearch = om.BoundsEnforceLS()
prob.model.nonlinear_solver = newton
prob.model.linear_solver = om.DirectSolver()
# sqlite file to record the intermediate calculations and derivatives
r = om.SqliteRecorder("pearl_computations.sql")
prob.add_recorder(r)
prob.driver.add_recorder(r)
prob.driver.recording_options["record_derivatives"] = True
# Attach recorder to a subsystem
model.nonlinear_solver.add_recorder(r)
model.add_recorder(r)
prob.driver.recording_options["includes"] = ["*"]
# Attach recorder to a solver
model.nonlinear_solver.add_recorder(r)
prob.setup()
prob.set_solver_print(level=2)
# For gradients across the model this will do the finite difference method
prob.model.approx_totals(method="fd", step=0.1, form="forward", step_calc="abs")
prob.run_model()
prob.run_driver()
prob.record("final_state")
print('minimum objective found at')
print(prob['cost_per_byte'][0])
print(prob['A_PV'])
print(f"tf: {prob['tf'][0]}")
results = dict()
results['tf'] = prob['tf'][0]
results['Df'] = prob['Df'][0]
results['ld'] = prob['ld'][0]
results['mass'] = prob['Payloadmass'][0]
results['DraughCon'] = prob['DraughtCon'][0]
results['AmplitudeCon'] = prob['AmplitudeCon'][0]
print(results)
Scaling report
I am trying to make a toy problem to learn a bit about the OpenMDAO software before applying the lessons to a larger problem. I have a problem set up so that the objective function should be minimized when both design variables are at a minimum. However both values stay at their originally assigned values despite receiving an 'Optimization terminated successfully' message.
I have been starting by writing the code based on the Sellar problem examples. ( http://openmdao.org/twodocs/versions/latest/basic_guide/sellar.html ) Additionally I have come across a stack overflow question that seems to be the same problem, but the solution there doesn't work. ( OpenMDAO: Solver converging to non-optimal point ) (When I add the declare_partials line to the IntermediateCycle or ScriptForTest I recieve an error saying either that self is not defined, or that the object has no attribute declare_partials)
This is the script that runs everything
import openmdao.api as om
from IntermediateForTest import IntermediateCycle
prob = om.Problem()
prob.model = IntermediateCycle()
prob.driver = om.ScipyOptimizeDriver()
#prob.driver.options['optimizer'] = 'SLSQP'
#prob.driver.options['tol'] = 1e-9
prob.model.add_design_var('n_gear', lower=2, upper=6)
prob.model.add_design_var('stroke', lower=0.0254, upper=1)
prob.model.add_objective('objective')
prob.setup()
prob.model.approx_totals()
prob.run_driver()
print(prob['objective'])
print(prob['cycle.f1.total_weight'])
print(prob['cycle.f1.stroke'])
print(prob['cycle.f1.n_gear'])
It calls an intermediate group, as per the Sellar example
import openmdao.api as om
from FunctionsForTest import FunctionForTest1
from FunctionsForTest import FunctionForTest2
class IntermediateCycle(om.Group):
def setup(self):
indeps = self.add_subsystem('indeps', om.IndepVarComp(), promotes=['*'])
indeps.add_output('n_gear', 3.0)
indeps.add_output('stroke', 0.2)
indeps.add_output('total_weight', 26000.0)
cycle = self.add_subsystem('cycle', om.Group())
cycle.add_subsystem('f1', FunctionForTest1())
cycle.add_subsystem('f2', FunctionForTest2())
cycle.connect('f1.landing_gear_weight','f2.landing_gear_weight')
cycle.connect('f2.total_weight','f1.total_weight')
self.connect('n_gear','cycle.f1.n_gear')
self.connect('stroke','cycle.f1.stroke')
#cycle.nonlinear_solver = om.NonlinearBlockGS()
self.nonlinear_solver = om.NonlinearBlockGS()
self.add_subsystem('objective', om.ExecComp('objective = total_weight', objective=26000, total_weight=26000), promotes=['objective', 'total_weight'])
Finally there is a file with the two functions in it:
import openmdao.api as om
class FunctionForTest1(om.ExplicitComponent):
def setup(self):
self.add_input('stroke', val=0.2)
self.add_input('n_gear', val=3.0)
self.add_input('total_weight', val=26000)
self.add_output('landing_gear_weight')
self.declare_partials('*', '*', method='fd')
def compute(self, inputs, outputs):
stroke = inputs['stroke']
n_gear = inputs['n_gear']
total_weight = inputs['total_weight']
outputs['landing_gear_weight'] = total_weight * 0.1 + 100*stroke * n_gear ** 2
class FunctionForTest2(om.ExplicitComponent):
def setup(self):
self.add_input('landing_gear_weight')
self.add_output('total_weight')
self.declare_partials('*', '*', method='fd')
def compute(self, inputs, outputs):
landing_gear_weight = inputs['landing_gear_weight']
outputs['total_weight'] = 26000 + landing_gear_weight
It reports optimization terminated successfully,
Optimization terminated successfully. (Exit mode 0)
Current function value: 26000.0
Iterations: 1
Function evaluations: 1
Gradient evaluations: 1
Optimization Complete
-----------------------------------
[26000.]
[29088.88888889]
[0.2]
[3.]
however the value for the function to optimize hasn't changed. It seems as it converges the loop to estimate the weight, but doesn't vary the design variables to find the optimum.
It arrives at 29088.9, which is correct for a value of n_gear=3 and stroke=0.2, but if both are decreased to the bounds of n_gear=2 and stroke=0.0254, it would arrive at a value of ~28900, ~188 less.
Any advice, links to tutorials, or solutions would be appreciated.
Lets take a look at the n2 of the model, as you provided it:
I've highlighted the connection from indeps.total_weight to objective.total_weight. So this means that your computed total_weight value is not being passed to your objective output at all. Instead you have a constant value being set there.
Now, taking a small step back, lets look at the computation of the objective itself:
self.add_subsystem('objective', om.ExecComp('objective = total_weight', objective=26000, total_weight=26000), promotes=['objective', 'total_weight'])
So this is an odd use of the ExecComp, because it just sets the output to exactly the input. It does nothing, and isn't really needed at all.
I believe what you wanted was simply to make the objective be the output f2.total_weight. When I do that (and make a few additional small cleanups to your code, like removing the unnecessary ExecComp, then I do get the correct answer in 2 major iterations of the optimizer:
import openmdao.api as om
class FunctionForTest1(om.ExplicitComponent):
def setup(self):
self.add_input('stroke', val=0.2)
self.add_input('n_gear', val=3.0)
self.add_input('total_weight', val=26000)
self.add_output('landing_gear_weight')
self.declare_partials('*', '*', method='fd')
def compute(self, inputs, outputs):
stroke = inputs['stroke']
n_gear = inputs['n_gear']
total_weight = inputs['total_weight']
outputs['landing_gear_weight'] = total_weight * 0.1 + 100*stroke * n_gear ** 2
class FunctionForTest2(om.ExplicitComponent):
def setup(self):
self.add_input('landing_gear_weight')
self.add_output('total_weight')
self.declare_partials('*', '*', method='fd')
def compute(self, inputs, outputs):
landing_gear_weight = inputs['landing_gear_weight']
outputs['total_weight'] = 26000 + landing_gear_weight
class IntermediateCycle(om.Group):
def setup(self):
indeps = self.add_subsystem('indeps', om.IndepVarComp(), promotes=['*'])
indeps.add_output('n_gear', 3.0)
indeps.add_output('stroke', 0.2)
cycle = self.add_subsystem('cycle', om.Group())
cycle.add_subsystem('f1', FunctionForTest1())
cycle.add_subsystem('f2', FunctionForTest2())
cycle.connect('f1.landing_gear_weight','f2.landing_gear_weight')
cycle.connect('f2.total_weight','f1.total_weight')
self.connect('n_gear','cycle.f1.n_gear')
self.connect('stroke','cycle.f1.stroke')
#cycle.nonlinear_solver = om.NonlinearBlockGS()
self.nonlinear_solver = om.NonlinearBlockGS()
prob = om.Problem()
prob.model = IntermediateCycle()
prob.driver = om.ScipyOptimizeDriver()
#prob.driver.options['optimizer'] = 'SLSQP'
#prob.driver.options['tol'] = 1e-9
prob.model.add_design_var('n_gear', lower=2, upper=6)
prob.model.add_design_var('stroke', lower=0.0254, upper=1)
prob.model.add_objective('cycle.f2.total_weight')
prob.model.approx_totals()
prob.setup()
prob.model.nl_solver.options['iprint'] = 2
prob.run_driver()
print(prob['cycle.f1.total_weight'])
print(prob['cycle.f2.total_weight'])
print(prob['cycle.f1.stroke'])
print(prob['cycle.f1.n_gear'])
gives:
Optimization terminated successfully. (Exit mode 0)
Current function value: 28900.177777779667
Iterations: 2
Function evaluations: 2
Gradient evaluations: 2
Optimization Complete
-----------------------------------
[28900.1777778]
[28900.17777778]
[0.0254]
[2.]
I am using sample 2D functions for optimization with MetaModelUnStructuredComp.
Below is a code snippet. The computational time spent for training increases considerably as I increase the number of sample points. I am not sure if this much increase is expected or am I doing something wrong.
The problem is 2D and predicting 1 output below is some performance time;
45 sec for 900 points*
14 sec for 625 points
3.7 sec for 400 points
*points represent the dimension of each training input
Will decreasing this be a focus of openMDAO development team in the future? (keep reading for the edited version)
import numpy as np
from openmdao.api import Problem, IndepVarComp
from openmdao.api import ScipyOptimizeDriver
from openmdao.api import MetaModelUnStructuredComp, FloatKrigingSurrogate,MetaModelUnStructuredComp
from openmdao.api import CaseReader, SqliteRecorder
import time
t0 = time.time()
class trig(MetaModelUnStructuredComp):
def setup(self):
ii=3
nx, ny = (10*ii, 10*ii)
print(nx*ny)
xx = np.linspace(-3,3, nx)
yy = np.linspace(-2,2, ny)
x, y = np.meshgrid(xx, yy)
# z = np.sin(x)**10 + np.cos(10 + y) * np.cos(x)
# z=4+4.5*x-4*y+x**2+2*y**2-2*x*y+x**4-2*x**2*y
term1 = (4-2.1*x**2+(x**4)/3) * x**2;
term2 = x*y;
term3 = (-4+4*y**2) * y**2;
z = term1 + term2 + term3;
self.add_input('x', training_data=x.flatten())
self.add_input('y', training_data=y.flatten())
self.add_output('meta_out', surrogate=FloatKrigingSurrogate(),
training_data=z.flatten())
prob = Problem()
inputs_comp = IndepVarComp()
inputs_comp.add_output('x', 1.5)
inputs_comp.add_output('y', 1.5)
prob.model.add_subsystem('inputs_comp', inputs_comp)
#triginst=
prob.model.add_subsystem('trig', trig())
prob.model.connect('inputs_comp.x', 'trig.x')
prob.model.connect('inputs_comp.y', 'trig.y')
prob.driver = ScipyOptimizeDriver()
prob.driver.options['optimizer'] = 'SLSQP'
prob.driver.options['tol'] = 1e-8
prob.driver.options['disp'] = True
prob.model.add_design_var('inputs_comp.x', lower=-3, upper=3)
prob.model.add_design_var('inputs_comp.y', lower=-2, upper=2)
prob.model.add_objective('trig.meta_out')
prob.setup(check=True)
prob.run_model()
print(prob['inputs_comp.x'])
print(prob['inputs_comp.y'])
print(prob['trig.meta_out'])
t1 = time.time()
total = t1-t0
print(total)
Following the answers below i am adding a code snippet of an explicit component that uses SMT toolbox for surrogate. I guess this is one way to use the toolbox's capabilities.
import numpy as np
from smt.surrogate_models import RBF
from openmdao.api import ExplicitComponent
from openmdao.api import Problem, ScipyOptimizeDriver
from openmdao.api import Group, IndepVarComp
import smt
# Sample problem with SMT Toolbox and OpenMDAO Explicit Comp
#Optimization of SIX-HUMP CAMEL FUNCTION with 2 global optima
class MetaCompSMT(ExplicitComponent):
def initialize(self):
self.options.declare('sm', types=smt.surrogate_models.rbf.RBF)
def setup(self):
self.add_input('x')
self.add_input('y')
self.add_output('z')
# self.declare_partials(of='z', wrt=['x','y'], method='fd')
self.declare_partials(of='*', wrt='*')
def compute(self, inputs, outputs):
# sm = self.options['sm'] # seems like this is not needed
sta=np.column_stack([inputs[i] for i in inputs])
outputs['z'] =sm.predict_values(sta).flatten()
def compute_partials(self, inputs, partials):
sta=np.column_stack([inputs[i] for i in inputs])
print(sta)
for i,invar in enumerate(inputs):
partials['z', invar] =sm.predict_derivatives(sta,i)
# SMT SURROGATE IS TRAINED IN ADVANCE AND PASSED TO THE COMPONENT AS GLOBAL INPUT
# Training Data
ii=3 # "incerases the domain size"
nx, ny = (10*ii, 5*ii)
x, y = np.meshgrid(np.linspace(-3,3, nx), np.linspace(-2,2, ny))
term1 = (4-2.1*x**2+(x**4)/3) * x**2;
term2 = x*y;
term3 = (-4+4*y**2) * y**2;
z = term1 + term2 + term3;
# Surrogate training
xt=np.column_stack([x.flatten(),y.flatten()])
yt=z.flatten()
#sm = KPLSK(theta0=[1e-2])
sm=RBF(d0=-1,poly_degree=-1,reg=1e-13,print_global=False)
sm.set_training_values(xt, yt)
sm.train()
prob = Problem() # Start the OpenMDAO optimization problem
prob.model = model = Group() # Assemble a group within the problem. In this case single group.
"Independent component ~ single Design variable "
inputs_comp = IndepVarComp() # OpenMDAO approach for the design variable as independent component output
inputs_comp.add_output('x', 2.5) # Vary initial value for finding the second global optimum
inputs_comp.add_output('y', 1.5) # Vary initial value for finding the second global optimum
model.add_subsystem('inputs_comp', inputs_comp)
"Component 1"
comp = MetaCompSMT(sm=sm)
model.add_subsystem('MetaCompSMT', comp)
"Connect design variable to the 2 components. Easier to follow than promote"
model.connect('inputs_comp.x', 'MetaCompSMT.x')
model.connect('inputs_comp.y', 'MetaCompSMT.y')
"Lower/Upper bound design variables"
model.add_design_var('inputs_comp.x', lower=-3, upper=3)
model.add_design_var('inputs_comp.y', lower=-2, upper=2)
model.add_objective('MetaCompSMT.z')
prob.driver = ScipyOptimizeDriver()
prob.driver.options['optimizer'] = 'SLSQP'
prob.driver.options['disp'] = True
prob.driver.options['tol'] = 1e-9
prob.setup(check=True, mode='fwd')
prob.run_driver()
print(prob['inputs_comp.x'],prob['inputs_comp.y'],prob['MetaCompSMT.z'])
If you are willing to compile some code yourself, you could write very light weight wrapper for the Surrogate Modeling Toolbox (SMT). You could write that wrapper to work with the standard MetaModelUnstructuredComp or just write your own component wrapper.
Either way, that library has some significantly faster unstructured surrogate models in it. The default OpenMDAO implementations are just basic implementations. We may improve them over time, but for larger data sets or design spaces SMT offers much better algorithms.
We haven't written a general SMT wrapper in OpenMDAO as of Version 2.4, but its not hard to write your own.
I'm going to look into the performance of the MetaModelUnStructuredComp using your test case a bit more closely. Though I do notice that this test case does involve fitting a structured data set. If you were to use MetaModelStructuredComp(http://openmdao.org/twodocs/versions/2.2.0/features/building_blocks/components/metamodelstructured.html), the performance is considerably better:
class trig(MetaModelStructuredComp):
def setup(self):
ii=3
nx, ny = (10*ii, 10*ii)
xx = np.linspace(-3,3, nx)
yy = np.linspace(-2,2, ny)
x, y = np.meshgrid(xx, yy, indexing='ij')
term1 = (4-2.1*x**2+(x**4)/3) * x**2;
term2 = x*y;
term3 = (-4+4*y**2) * y**2;
z = term1 + term2 + term3;
self.add_input('x', 0.0, xx)
self.add_input('y', 0.0, yy)
self.add_output('meta_out', 0.0, z)
The 900 points case goes from 14 seconds on my machine using MetaModelUnStructuredComp to 0.081 when using MetaModelStructuredComp.
I am using three-dimensional convolution links (with ConvolutionND) in my chain.
The forward computation run smoothly (I checked intermediate result shapes to be sure I understood correctly the meaning of the parameters of convolution_nd), but during the backward a CuDNNError is raised with the message CUDNN_STATUS_NOT_SUPPORTED.
The cover_all parameter of ConvolutionND as its default value of False, so from the doc I don't see what can be the cause of the error.
Here is how I defind one of the convolution layers :
self.conv1 = chainer.links.ConvolutionND(3, 1, 4, (3, 3, 3)).to_gpu(self.GPU_1_ID)
And the call stack is
File "chainer/function_node.py", line 548, in backward_accumulate
gxs = self.backward(target_input_indexes, grad_outputs)
File "chainer/functions/connection/convolution_nd.py", line 118, in backward
gy, W, stride=self.stride, pad=self.pad, outsize=x_shape)
File "chainer/functions/connection/deconvolution_nd.py", line 310, in deconvolution_nd
y, = func.apply(args)
File chainer/function_node.py", line 258, in apply
outputs = self.forward(in_data)
File "chainer/functions/connection/deconvolution_nd.py", line 128, in forward
return self._forward_cudnn(x, W, b)
File "chainer/functions/connection/deconvolution_nd.py", line 105, in _forward_cudnn
tensor_core=tensor_core)
File "cupy/cudnn.pyx", line 881, in cupy.cudnn.convolution_backward_data
File "cupy/cuda/cudnn.pyx", line 975, in cupy.cuda.cudnn.convolutionBackwardData_v3
File "cupy/cuda/cudnn.pyx", line 461, in cupy.cuda.cudnn.check_status
cupy.cuda.cudnn.CuDNNError: CUDNN_STATUS_NOT_SUPPORTED
So are there special points to take care of when using ConvolutionND ?
A failing code is for instance :
import chainer
from chainer import functions as F
from chainer import links as L
from chainer.backends import cuda
import numpy as np
import cupy as cp
chainer.global_config.cudnn_deterministic = False
NB_MASKS = 60
NB_FCN = 3
NB_CLASS = 17
class MFEChain(chainer.Chain):
"""docstring for Wavelphasenet."""
def __init__(self,
FCN_Dim,
gpu_ids=None):
super(MFEChain, self).__init__()
self.GPU_0_ID, self.GPU_1_ID = (0, 1) if gpu_ids is None else gpu_ids
with self.init_scope():
self.conv1 = chainer.links.ConvolutionND(3, 1, 4, (3, 3, 3)).to_gpu(
self.GPU_1_ID
)
def __call__(self, inputs):
### Pad input ###
processed_sequences = []
for convolved in inputs:
## Transform to sequences)
copy = convolved if self.GPU_0_ID == self.GPU_1_ID else F.copy(convolved, self.GPU_1_ID)
processed_sequences.append(copy)
reprocessed_sequences = []
with cuda.get_device(self.GPU_1_ID):
for convolved in processed_sequences:
convolved = F.expand_dims(convolved, 0)
convolved = F.expand_dims(convolved, 0)
convolved = self.conv1(convolved)
reprocessed_sequences.append(convolved)
states = F.vstack(reprocessed_sequences)
logits = states
ret_logits = logits if self.GPU_0_ID == self.GPU_1_ID else F.copy(logits, self.GPU_0_ID)
return ret_logits
def mfe_test():
mfe = MFEChain(150)
inputs = list(
chainer.Variable(
cp.random.randn(
NB_MASKS,
11,
in_len,
dtype=cp.float32
)
) for in_len in [53248]
)
val = mfe(inputs)
grad = cp.ones(val.shape, dtype=cp.float32)
val.grad = grad
val.backward()
for i in inputs:
print(i.grad)
if __name__ == "__main__":
mfe_test()
cupy.cuda.cudnn.convolutionBackwardData_v3 is incompatible with some specific parameters, as described in an issue in official github.
Unfortunately, the issue only dealt with deconvolution_2d.py (not deconvolution_nd.py), therefore the decision-making about whether cudnn is used or not failed in your case, I guess.
you can check your parameter by confirming
check whether dilation parameter (!=1) or group parameter (!=1) is passed to the convolution.
print chainer.config.cudnn_deterministic, configuration.config.autotune, and configuration.config.use_cudnn_tensor_core.
Further support may be obtained by raising an issue in the official github.
The code you showed is much complicated.
To clarify the problem, the code below would help.
from chainer import Variable, Chain
from chainer import links as L
from chainer import functions as F
import numpy as np
from six import print_
batch_size = 1
in_channel = 1
out_channel = 1
class MyLink(Chain):
def __init__(self):
super(MyLink, self).__init__()
with self.init_scope():
self.conv = L.ConvolutionND(3, 1, 1, (3, 3, 3), nobias=True, initialW=np.ones((in_channel, out_channel, 3, 3, 3)))
def __call__(self, x):
return F.sum(self.conv(x))
if __name__ == "__main__":
my_link = MyLink()
my_link.to_gpu(0)
batch = Variable(np.ones((batch_size, in_channel, 3, 3, 3)))
batch.to_gpu(0)
loss = my_link(batch)
loss.backward()
print_(batch.grad)
I have a param that is a 2D array. It works fine with getting the correct output but when I try to do anything with the gradients such as optimization or check_total_derivatives I get a sizing error. I was wondering what the best way is to handle params that are of size 2D. Here is a sample code:
import numpy as np
from openmdao.api import Group, Problem, Component, IndepVarComp, ExecComp
class C1(Component):
def __init__(self, n):
super(C1, self).__init__()
self.add_param('grid', val=np.zeros((n, n)))
self.add_output('x', shape=1)
self.n = n
def solve_nonlinear(self, params, unknowns, resids):
x = 0
for i in range(self.n):
for j in range(self.n):
x += params['grid'][i][j]
unknowns['x'] = x
def linearize(self, params, unknowns, resids):
J = {}
J['x', 'grid'] = np.ones((self.n, self.n))
return J
class Group1(Group):
def __init__(self, n):
super(Group1, self).__init__()
self.add('grid', IndepVarComp('grid', np.zeros((n, n))), promotes=['*'])
self.add('c1', C1(n), promotes=['*'])
self.add('obj_cmp', ExecComp('obj = -x', x=1.0), promotes=['*'])
n = 3
p = Problem()
p.root = Group1(n)
p.setup(check=False)
p['grid'] = np.ones((n, n))
p.run()
p.check_total_derivatives()
print p['x']
I get the error:
ValueError: In component 'c1', the derivative of 'x' wrt 'grid' should have shape '(1, 3)' but has shape '(3, 3)' instead.
I feel like the derivative in this case should be of size (3, 3) because that is the size of the input param. How do you handle 2D params?
You have a small mistake in the Jacobian; it should look like this:
def linearize(self, params, unknowns, resids):
J = {}
J['x', 'grid'] = np.ones((1, self.n*self.n))
return J
The output x is length 1, while the param grid is n by n, so it is length n*n, so the resulting J should be 1 by 9. With that change, I get the right answer.
I did notice a mistake in the error message. It should say that the expected shape is (1, 9) instead of (1, 3). I will put in a fix for that.
When you have a 2D variable and need to construct the gradient, flatten it (in row-major order) and formulate the gradient based on the flattened version.