AIRFLOW - "Oops Something bad has happened" - airflow

Everything was working just fine with my airflow fresh install, until I got this "Oops something bad has happened" message again. Had it previously when editing the connections.
Traceback (most recent call last):
File "/home/mica/.local/lib/python3.8/site-packages/flask/app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "/home/mica/.local/lib/python3.8/site-packages/flask/app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/home/mica/.local/lib/python3.8/site-packages/flask/app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/home/mica/.local/lib/python3.8/site-packages/flask/_compat.py", line 39, in reraise
raise value
File "/home/mica/.local/lib/python3.8/site-packages/flask/app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "/home/mica/.local/lib/python3.8/site-packages/flask/app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/home/mica/.local/lib/python3.8/site-packages/airflow/www/auth.py", line 51, in decorated
return func(*args, **kwargs)
File "/home/mica/.local/lib/python3.8/site-packages/airflow/www/views.py", line 718, in index
paging=wwwutils.generate_pages(
File "/home/mica/.local/lib/python3.8/site-packages/airflow/www/utils.py", line 107, in generate_pages
first_node = Markup(
File "/home/mica/.local/lib/python3.8/site-packages/jinja2/utils.py", line 838, in __new__
warnings.warn(
File "/usr/lib/python3.8/warnings.py", line 109, in _showwarnmsg
sw(msg.message, msg.category, msg.filename, msg.lineno,
File "/home/mica/.local/lib/python3.8/site-packages/airflow/settings.py", line 117, in custom_show_warning
write_console.print(msg, soft_wrap=True)
File "/home/mica/.local/lib/python3.8/site-packages/rich/console.py", line 1642, in print
self._buffer.extend(new_segments)
File "/home/mica/.local/lib/python3.8/site-packages/rich/console.py", line 842, in __exit__
self._exit_buffer()
File "/home/mica/.local/lib/python3.8/site-packages/rich/console.py", line 800, in _exit_buffer
self._check_buffer()
File "/home/mica/.local/lib/python3.8/site-packages/rich/console.py", line 1934, in _check_buffer
self.file.write(text)
OSError: [Errno 5] Input/output error

The problem happened when I was launching airflow from the home. If you navigate to where the application is and then start the webserver, it worked for me.

Related

OperationalError: attempt to write a readonly DB on Vercel via Github (the DB and its folder are full-access on my local comp)

I am trying to deploy a simple Flask app with a SQLite database. It works fine when running locally. I deploy it on Vercel via Github. It works until it needs to write to the db. Then there is
OperationalError: attempt to write a readonly database
Both the DB and the folder ‘instance’ that contains it are Full-access on my local system, I can see it with
attrib ~/instance
in the command line (I am working with Windows 10). Is there a way to check and change the access mode of a file or a folder right on the Github repo?
Here are the logs regarding this error:
[ERROR] 2023-01-12T17:19:33.931Z 63bacb5b-ec13-4247-b784-9aaba99376a0
Exception on /register [POST]Traceback (most recent call last):
File "/var/task/sqlalchemy/engine/base.py", line 1900, in _execute_context self.dialect.do_execute
( File "/var/task/sqlalchemy/engine/default.py", line 736, in do_execute cursor.execute(statement, parameters)
sqlite3.OperationalError: attempt to write a readonly database
The above exception was the direct cause of the following exception:
Traceback (most recent call last): File "/var/task/flask/app.py", line 2525, in wsgi_app response = self.full_dispatch_request() File "/var/task/flask/app.py", line 1822, in full_dispatch_request rv = self.handle_user_exception(e) File "/var/task/flask/app.py", line 1820, in full_dispatch_request rv = self.dispatch_request() File "/var/task/flask/app.py", line 1796, in dispatch_request return self.ensure_sync(self.view_functions[rule.endpoint])(**view_args) File "./app.py", line 136, in register db.session.commit()
File "<string>", line 2, in commit File "/var/task/sqlalchemy/orm/session.py", line 1451, in commit self._transaction.commit(_to_root=self.future) File "/var/task/sqlalchemy/orm/session.py", line 829, in commit self._prepare_impl() File "/var/task/sqlalchemy/orm/session.py", line 808, in _prepare_impl self.session.flush() File "/var/task/sqlalchemy/orm/session.py", line 3444, in flush self._flush(objects) File "/var/task/sqlalchemy/orm/session.py", line 3584, in _flush transaction.rollback(_capture_exception=True) File "/var/task/sqlalchemy/util/langhelpers.py", line 70, in __exit__ compat.raise_( File "/var/task/sqlalchemy/util/compat.py", line 211, in raise_ raise exception File "/var/task/sqlalchemy/orm/session.py", line 3544, in _flush flush_context.execute() File "/var/task/sqlalchemy/orm/unitofwork.py", line 456, in execute rec.execute(self) File "/var/task/sqlalchemy/orm/unitofwork.py", line 630, in execute util.preloaded.orm_persistence.save_obj( File "/var/task/sqlalchemy/orm/persistence.py", line 245, in save_obj _emit_insert_statements( File "/var/task/sqlalchemy/orm/persistence.py", line 1238, in _emit_insert_statements result = connection._execute_20( File "/var/task/sqlalchemy/engine/base.py", line 1705, in _execute_20 return meth(self, args_10style, kwargs_10style, execution_options) File "/var/task/sqlalchemy/sql/elements.py", line 334, in _execute_on_connection return connection._execute_clauseelement( File "/var/task/sqlalchemy/engine/base.py", line 1572, in _execute_clauseelement ret = self._execute_context( File "/var/task/sqlalchemy/engine/base.py", line 1943, in _execute_context self._handle_dbapi_exception( File "/var/task/sqlalchemy/engine/base.py", line 2124, in _handle_dbapi_exception util.raise_( File "/var/task/sqlalchemy/util/compat.py", line 211, in raise_ raise exception File "/var/task/sqlalchemy/engine/base.py", line 1900, in _execute_context self.dialect.do_execute( File "/var/task/sqlalchemy/engine/default.py", line 736, in do_execute cursor.execute(statement, parameters)sqlalchemy.exc.
OperationalError: (sqlite3.OperationalError) attempt to write a readonly database
I double-checked the access mode of the DB in my local system both as User and as Admin
It turns out that Vercel does not support SQLite. It is clearly written in their guides:
SQLite needs a local file system on the server to store the data
permanently when write requests are made. In a serverless environment,
this central single permanent storage is not available because storage
is ephemeral with serverless functions.
So it's not the problem of a Read-only mode, but rather the issue with SQL databases on Vercel.

Airflow Web UI LDAP authentication throws AttributeError: 'NoneType' object has no attribute 'username

I have configured LDAP authentication for airflow, managed in OpenShift.
If I access the web UI, without clearing cache, after a redeployment, it throws the following error in web server.
It works after the browser cache is cleared.
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/sync.py", line 134, in handle
self.handle_request(listener, req, client, addr)
File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/sync.py", line 175, in handle_request
respiter = self.wsgi(environ, resp.start_response)
File "/usr/local/lib/python3.7/site-packages/werkzeug/middleware/dispatcher.py", line 66, in __call__
return app(environ, start_response)
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2464, in __call__
return self.wsgi_app(environ, start_response)
File "/usr/local/lib/python3.7/site-packages/werkzeug/middleware/proxy_fix.py", line 232, in __call__
return self.app(environ, start_response)
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2450, in wsgi_app
response = self.handle_exception(e)
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1879, in handle_exception
server_error = handler(server_error)
File "/usr/local/lib/python3.7/site-packages/airflow/www/views.py", line 776, in show_traceback
fallback=True) else 'Error! Please contact server admin'), 500
File "/usr/local/lib/python3.7/site-packages/flask/templating.py", line 136, in render_template
ctx.app.update_template_context(context)
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 838, in update_template_context
context.update(func())
File "/usr/local/lib/python3.7/site-packages/flask_login/utils.py", line 368, in _user_context_processor
return dict(current_user=_get_user())
File "/usr/local/lib/python3.7/site-packages/flask_login/utils.py", line 335, in _get_user
current_app.login_manager._load_user()
File "/usr/local/lib/python3.7/site-packages/flask_login/login_manager.py", line 359, in _load_user
return self.reload_user()
File "/usr/local/lib/python3.7/site-packages/flask_login/login_manager.py", line 321, in reload_user
user = self.user_callback(user_id)
File "/usr/local/lib/python3.7/site-packages/airflow/utils/db.py", line 74, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/airflow/contrib/auth/backends/ldap_auth.py", line 280, in load_user
return LdapUser(user)
File "<string>", line 4, in __init__
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/orm/state.py", line 433, in _initialize_instance
manager.dispatch.init_failure(self, args, kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 69, in __exit__
exc_value, with_traceback=exc_tb,
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 182, in raise_
raise exception
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/orm/state.py", line 430, in _initialize_instance
return manager.original_init(*mixed[1:], **kwargs)
File "/usr/local/lib/python3.7/site-packages/airflow/contrib/auth/backends/ldap_auth.py", line 184, in __init__
user.username
AttributeError: 'NoneType' object has no attribute 'username
Am I missing out anything in airflow.cfg?

cannot enable nbextensions in Jupyter Notebook with new Conda Environment on IBM DSX

Trying to install this package:-
https://github.com/lspvic/jupyter_tensorboard
Python 2.7
I cannot enable nbextensions in Jupyter Notebook with new Conda Environment
Here is the stacktrace
Enabling notebook extension jupyter_tensorboard/tree...
Traceback (most recent call last):
File "/opt/conda/envs/DSX-Python27/bin/jupyter-nbextension", line 11, in <module>
sys.exit(main())
File "/opt/conda/envs/DSX-Python27/lib/python2.7/site-packages/jupyter_core/application.py", line 267, in launch_instance
return super(JupyterApp, cls).launch_instance(argv=argv, **kwargs)
File "/opt/conda/envs/DSX-Python27/lib/python2.7/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/opt/conda/envs/DSX-Python27/lib/python2.7/site-packages/notebook/nbextensions.py", line 988, in start
super(NBExtensionApp, self).start()
File "/opt/conda/envs/DSX-Python27/lib/python2.7/site-packages/jupyter_core/application.py", line 256, in start
self.subapp.start()
File "/opt/conda/envs/DSX-Python27/lib/python2.7/site-packages/notebook/nbextensions.py", line 898, in start
self.toggle_nbextension(self.extra_args[0])
File "/opt/conda/envs/DSX-Python27/lib/python2.7/site-packages/notebook/nbextensions.py", line 888, in toggle_nbextension
logger=self.log)
File "/opt/conda/envs/DSX-Python27/lib/python2.7/site-packages/notebook/nbextensions.py", line 413, in enable_nbextension
logger=logger)
File "/opt/conda/envs/DSX-Python27/lib/python2.7/site-packages/notebook/nbextensions.py", line 351, in _set_nbextension_state
cm.update(section, {"load_extensions": {require: state}})
File "/opt/conda/envs/DSX-Python27/lib/python2.7/site-packages/traitlets/config/manager.py", line 87, in update
self.set(section_name, data)
File "/opt/conda/envs/DSX-Python27/lib/python2.7/site-packages/traitlets/config/manager.py", line 71, in set
self.ensure_config_dir_exists()
File "/opt/conda/envs/DSX-Python27/lib/python2.7/site-packages/traitlets/config/manager.py", line 46, in ensure_config_dir_exists
os.makedirs(self.config_dir, 0o755)
File "/opt/conda/envs/DSX-Python27/lib/python2.7/os.py", line 157, in makedirs
mkdir(name, mode)
OSError: [Errno 13] Permission denied: '/opt/ibm/dsx/config/nbconfig'
Looks like permission error, Can it be install/enabled with user flag.

Flask/uWSGI/MariaDB running under Supervisor stops working almost daily

I'm running a Flask app via uWSGI which is managed by Supervisor (and the webserver is using Nginx).
The whole setup seems to work perfectly, however, I notice that every day or two, my flask app starts throwing 500 errors (which means there is a Sqlalchemy db.commit() error) and I can't recover until I go in and reload uwsgi, after which, everything works perfectly again. To reload, I typically kill the uwsgi processes, and supervisor brings them back in and it's fine. Once I've gotten it to work by just 'touch'ing the wsgi file.
Also strangely, whenever I run supervisor with processes=1, it automatically loads TWO uwsgi instances. The first PID is the one logged as being operated by supervisor, and the other doesn't register anywhere. However, regardless of this, killing the 2nd process - it always comes back.
Below is my uwsgi startup command run by supervisor
[program:{{ app_name }}]
command=/usr/local/bin/uwsgi
--socket=/tmp/{{ app_name }}.sock
--logto={{ log_dir }}/application.log
--home={{ site_dir }}/env
--pythonpath={{ site_dir }}
--virtualenv={{ env_path }}
--wsgi-file={{ webapps_dir }}/manage.py
--callable={{ wsgi_callable }}
--max-requests=1000
--master
--processes=1
--uid={{ deployment_user }}
--gid=www-data
--chmod-socket=664
--chown-socket={{ deployment_user }}:www-data
directory={{ webapps_dir }}
autostart=true
autorestart=true
EDIT: Here is the error that's spit out from the server.
2015-03-19 15:03:38,389 ERROR: Exception on /api/v1/account/login [POST] [in /var/www/website.com/env/local/lib/python2.7/site-packages/flask/app.py:1423]
Traceback (most recent call last):
File "/var/www/website.com/env/local/lib/python2.7/site-packages/flask/app.py", line 1817, in wsgi_app
response = self.full_dispatch_request()
File "/var/www/website.com/env/local/lib/python2.7/site-packages/flask/app.py", line 1477, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/flask_cors.py", line 272, in wrapped_function
return cors_after_request(app.make_response(f(*args, **kwargs)))
File "/var/www/website.com/env/local/lib/python2.7/site-packages/flask/app.py", line 1381, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/flask/app.py", line 1475, in full_dispatch_request
rv = self.dispatch_request()
File "/var/www/website.com/env/local/lib/python2.7/site-packages/flask/app.py", line 1461, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/flask_classy.py", line 200, in proxy
response = view(**request.view_args)
File "./webportal/account/api.py", line 34, in login_user
user = auth.authenticate(username=email, password=password)
File "/var/www/website.com/server/webportal/auth.py", line 23, in authenticate
user = User.query.filter_by(email=username).first()
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2367, in first
ret = list(self[0:1])
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2228, in __getitem__
return list(res)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2438, in __iter__
return self._execute_and_instances(context)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2453, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 729, in execute
return meth(self, multiparams, params)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 322, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauseelement
compiled_sql, distilled_params
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 958, in _execute_context
context)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1159, in _handle_dbapi_exception
exc_info
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 951, in _execute_context
context)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 436, in do_execute
cursor.execute(statement, parameters)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/MySQLdb/cursors.py", line 205, in execute
self.errorhandler(self, exc, value)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
OperationalError: (OperationalError) (2006, 'MySQL server has gone away') 'SELECT `Users`.id AS `Users_id`, `Users`.email AS `Users_email`, `Users`.password AS `Users_password`, `Users`.`emailConfirmed` AS `Users_emailConfirmed`, `Users`.`createdAtUtc` AS `Users_createdAtUtc`, `Users`.`updatedAtUtc` AS `Users_updatedAtUtc`, `Users`.`roleId` AS `Users_roleId` \nFROM `Users` \nWHERE `Users`.email = %s \n LIMIT %s' ('user#website.demo', 1)
2015-03-19 15:06:22,089 ERROR: Exception on /api/v1/account/login [POST] [in /var/www/website.com/env/local/lib/python2.7/site-packages/flask/app.py:1423]
Traceback (most recent call last):
File "/var/www/website.com/env/local/lib/python2.7/site-packages/flask/app.py", line 1817, in wsgi_app
response = self.full_dispatch_request()
File "/var/www/website.com/env/local/lib/python2.7/site-packages/flask/app.py", line 1477, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/flask_cors.py", line 272, in wrapped_function
return cors_after_request(app.make_response(f(*args, **kwargs)))
File "/var/www/website.com/env/local/lib/python2.7/site-packages/flask/app.py", line 1381, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/flask/app.py", line 1475, in full_dispatch_request
rv = self.dispatch_request()
File "/var/www/website.com/env/local/lib/python2.7/site-packages/flask/app.py", line 1461, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/flask_classy.py", line 200, in proxy
response = view(**request.view_args)
File "./webportal/account/api.py", line 34, in login_user
user = auth.authenticate(username=email, password=password)
File "/var/www/website.com/server/webportal/auth.py", line 23, in authenticate
user = User.query.filter_by(email=username).first()
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2367, in first
ret = list(self[0:1])
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2228, in __getitem__
return list(res)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2438, in __iter__
return self._execute_and_instances(context)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2453, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 729, in execute
return meth(self, multiparams, params)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 322, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauseelement
compiled_sql, distilled_params
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 893, in _execute_context
None, None)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1159, in _handle_dbapi_exception
exc_info
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 887, in _execute_context
conn = self._revalidate_connection()
File "/var/www/website.com/env/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 242, in _revalidate_connection
"Can't reconnect until invalid "
StatementError: Can't reconnect until invalid transaction is rolled back (original cause: InvalidRequestError: Can't reconnect until invalid transaction is rolled back) u'SELECT `Users`.id AS `Users_id`, `Users`.email AS `Users_email`, `Users`.password AS `Users_password`, `Users`.`emailConfirmed` AS `Users_emailConfirmed`, `Users`.`createdAtUtc` AS `Users_createdAtUtc`, `Users`.`updatedAtUtc` AS `Users_updatedAtUtc`, `Users`.`roleId` AS `Users_roleId` \nFROM `Users` \nWHERE `Users`.email = %s \n LIMIT %s' [immutabledict({})]
According to https://dev.mysql.com/doc/refman/5.0/en/gone-away.html, MySQL closes the connection after 8 hours... Once that happens, the app gets in a bit of a state.

DevStack -- Unable to lauch instance due to nova-rootwrap

Message
Remote error: ProcessExecutionError Unexpected error while running command. Command: sudo nova-rootwrap /etc/nova/rootwrap.conf env CONFIG_FILE=["/etc/nova/nova.conf"] NETWORK_ID=1 dnsmasq --strict-order --bind-interfaces --conf-file= --pid-file=/opt/stac
Code
500
Details
File "/opt/stack/nova/nova/compute/manager.py", line 293, in decorated_function return function(self, context, *args, **kwargs) File "/opt/stack/nova/nova/compute/manager.py", line 2003, in run_instance do_run_instance() File "/opt/stack/nova/nova/openstack/common/lockutils.py", line 249, in inner return f(*args, **kwargs) File "/opt/stack/nova/nova/compute/manager.py", line 2002, in do_run_instance legacy_bdm_in_spec) File "/opt/stack/nova/nova/compute/manager.py", line 1150, in _run_instance notify("error", fault=e) # notify that build failed File "/opt/stack/nova/nova/openstack/common/excutils.py", line 68, in __exit__ six.reraise(self.type_, self.value, self.tb) File "/opt/stack/nova/nova/compute/manager.py", line 1134, in _run_instance instance, image_meta, legacy_bdm_in_spec) File "/opt/stack/nova/nova/compute/manager.py", line 1287, in _build_instance filter_properties, bdms, legacy_bdm_in_spec) File "/opt/stack/nova/nova/compute/manager.py", line 1333, in _reschedule_or_error self._log_original_error(exc_info, instance_uuid) File "/opt/stack/nova/nova/openstack/common/excutils.py", line 68, in __exit__ six.reraise(self.type_, self.value, self.tb) File "/opt/stack/nova/nova/compute/manager.py", line 1328, in _reschedule_or_error bdms, requested_networks) File "/opt/stack/nova/nova/compute/manager.py", line 2055, in _shutdown_instance self._try_deallocate_network(context, instance, requested_networks) File "/opt/stack/nova/nova/compute/manager.py", line 2014, in _try_deallocate_network self._set_instance_error_state(context, instance['uuid']) File "/opt/stack/nova/nova/openstack/common/excutils.py", line 68, in __exit__ six.reraise(self.type_, self.value, self.tb) File "/opt/stack/nova/nova/compute/manager.py", line 2009, in _try_deallocate_network self._deallocate_network(context, instance, requested_networks) File "/opt/stack/nova/nova/compute/manager.py", line 1708, in _deallocate_network context, instance, requested_networks=requested_networks) File "/opt/stack/nova/nova/network/api.py", line 94, in wrapped return func(self, context, *args, **kwargs) File "/opt/stack/nova/nova/network/api.py", line 318, in deallocate_for_instance requested_networks=requested_networks) File "/opt/stack/nova/nova/network/rpcapi.py", line 190, in deallocate_for_instance return cctxt.call(ctxt, 'deallocate_for_instance', **kwargs) File "/opt/stack/oslo.messaging/oslo/messaging/rpc/client.py", line 361, in call return self.prepare().call(ctxt, method, **kwargs) File "/opt/stack/oslo.messaging/oslo/messaging/rpc/client.py", line 150, in call wait_for_reply=True, timeout=timeout) File "/opt/stack/oslo.messaging/oslo/messaging/transport.py", line 90, in _send timeout=timeout) File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/amqpdriver.py", line 409, in send return self._send(target, ctxt, message, wait_for_reply, timeout) File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/amqpdriver.py", line 402, in _send raise result
I am not able to figure out the problem. Please help!
I made change in localrc file as below and run stack.sh script again, it worked:
stack#ubuntu:~/devstack$ cat localrc
FLOATING_RANGE=192.168.1.224/27
FIXED_RANGE=10.11.12.0/24
FIXED_NETWORK_SIZE=256
ADMIN_PASSWORD=root123
MYSQL_PASSWORD=root123
RABBIT_PASSWORD=root123
SERVICE_PASSWORD=root123
FORCE=yes
SERVICE_TOKEN=root123
LOGFILE=/opt/stack/logs/stack.sh.log
FLOATING_RANGE=192.168.1.224/27
FIXED_RANGE=10.11.12.0/24
FIXED_NETWORK_SIZE=256
FLAT_INTERFACE=eth0
stack#ubuntu:~/devstack$

Resources