Hi I am using biopython to pull files from NCBI using Entrez. The program works on small files but on larger files I get an error. I would really appreciate some insight or help figuring out what went wrong.
Here is the program:
from Bio import Entrez
Entrez.email = "jbro262#lsu.edu"
search_handle = Entrez.esearch(db="nucleotide",term="Saimiri",usehistory="n")
search_results = Entrez.read(search_handle)
search_handle.close()
gi_list = search_results["IdList"]
count = int(search_results["Count"])
a = open("Numfile.txt", "a+")
a.write("The number of Saimiri files are :")
a.write(str(count))
a.write("\n")
a.close()
webenv = search_results["WebEnv"]
query_key = search_results["QueryKey"]
batch_size = 25
out_handle = open("SaimiriDNA.fasta", "w")
for start in range(0,count,batch_size):
end = min(count, start+batch_size)
print("Going to download record %i to %i" % (start+1, end))
fetch_handle = Entrez.efetch(db="nucleotide", rettype="fasta", retmode="text", retstart=start, retmax=batch_size, webenv=webenv, query_key=query_key)
data=fetch_handle.read()
fetch_handle.close()
out_handle.write(data)
out_handle.close()
Here are the errors:
Traceback (most recent call last):
File "Entrezfiles_Saimiri.py", line 53, in <module>
fetch_handle = Entrez.efetch(db="nucleotide", rettype="fasta", retmode="text", retstart=start, retmax=batch_size, webenv=webenv, query_key=query_key)
File "/usr/local/lib/python3.4/
dist-packages/Bio/Entrez/__init__.py", line 149, in efetch
return _open(cgi, variables, post)
File "/usr/local/lib/python3.4/dist-packages/Bio/Entrez/__init__.py", line 464, in _open
raise exception
File "/usr/local/lib/python3.4/dist-packages/Bio/Entrez/__init__.py", line 462, in _open
handle = _urlopen(cgi)
File "/usr/lib/python3.4/urllib/request.py", line 153, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python3.4/urllib/request.py", line 461, in open
response = meth(req, response)
File "/usr/lib/python3.4/urllib/request.py", line 571, in http_response
'http', request, response, code, msg, hdrs)
File "/usr/lib/python3.4/urllib/request.py", line 499, in error
return self._call_chain(*args)
File "/usr/lib/python3.4/urllib/request.py", line 433, in _call_chain
result = func(*args)
File "/usr/lib/python3.4/urllib/request.py", line 579, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 502: Bad Gateway
Does this mean something is going wrong with my server while the files are downloading?
Related
Hi I have encountered such problem when loading the function install_tensorflow() in r.
I don't know what to do with this error. Any help will be greatly appreciated!
Below is the error message
ERROR: Exception:
Traceback (most recent call last):
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_vendor/resolvelib/resolvers.py", line 171, in _merge_into_criterion
crit = self.state.criteria[name]
KeyError: 'tensorflow'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_vendor/urllib3/response.py", line 438, in _error_catcher
yield
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_vendor/urllib3/response.py", line 519, in read
data = self._fp.read(amt) if not fp_closed else b""
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_vendor/cachecontrol/filewrapper.py", line 62, in read
data = self.__fp.read(amt)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/http/client.py", line 461, in read
n = self.readinto(b)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/http/client.py", line 505, in readinto
n = self.fp.readinto(b)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/ssl.py", line 1071, in recv_into
return self.read(nbytes, buffer)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/ssl.py", line 929, in read
return self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/cli/base_command.py", line 189, in _main
status = self.run(options, args)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/cli/req_command.py", line 178, in wrapper
return func(self, options, args)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/commands/install.py", line 317, in run
reqs, check_supported_wheels=not options.target_dir
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/resolver.py", line 122, in resolve
requirements, max_rounds=try_to_avoid_resolution_too_deep,
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_vendor/resolvelib/resolvers.py", line 453, in resolve
state = resolution.resolve(requirements, max_rounds=max_rounds)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_vendor/resolvelib/resolvers.py", line 318, in resolve
name, crit = self._merge_into_criterion(r, parent=None)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_vendor/resolvelib/resolvers.py", line 173, in _merge_into_criterion
crit = Criterion.from_requirement(self._p, requirement, parent)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_vendor/resolvelib/resolvers.py", line 82, in from_requirement
if not cands:
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_vendor/resolvelib/structs.py", line 124, in bool
return bool(self._sequence)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 143, in bool
return any(self)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 38, in _iter_built
candidate = func()
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/factory.py", line 169, in _make_candidate_from_link
name=name, version=version,
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 306, in init
version=version,
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 144, in init
self.dist = self._prepare()
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 226, in _prepare
dist = self._prepare_distribution()
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 312, in _prepare_distribution
self._ireq, parallel_builds=True,
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/operations/prepare.py", line 457, in prepare_linked_requirement
return self._prepare_linked_requirement(req, parallel_builds)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/operations/prepare.py", line 482, in _prepare_linked_requirement
self.download_dir, hashes,
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/operations/prepare.py", line 234, in unpack_url
hashes=hashes,
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/operations/prepare.py", line 108, in get_http_url
from_path, content_type = download(link, temp_dir.path)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/network/download.py", line 163, in call
for chunk in chunks:
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/cli/progress_bars.py", line 159, in iter
for x in it:
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_internal/network/utils.py", line 88, in response_chunks
decode_content=False,
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_vendor/urllib3/response.py", line 576, in stream
data = self.read(amt=amt, decode_content=decode_content)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_vendor/urllib3/response.py", line 541, in read
raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/contextlib.py", line 130, in exit
self.gen.throw(type, value, traceback)
File "/Users/alexsong/Library/r-miniconda/envs/r-reticulate/lib/python3.7/site-packages/pip/_vendor/urllib3/response.py", line 443, in _error_catcher
raise ReadTimeoutError(self._pool, None, "Read timed out.")
pip._vendor.urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='files.pythonhosted.org', port=443): Read timed out.
Error: Error installing package(s): 'tensorflow==2.4'
I'm having some problems with my flask application which I never had before until today and I haven't changed anything so it's quite confusing
So it my app.py
#app.route('/view.html')
def data():
try:
path = 'app/database.db'
con = sqlite3.connect(path)
cur = con.cursor()
con.row_factory = sqlite3.Row
rows = cur.execute("select * from table").fetchall()
return render_template('layouts/default.html',
content=render_template('pages/view.html',
rows=rows))
except sqlite3.OperationalError:
return render_template('layouts/default.html',
content=render_template('pages/view.html'))
The idea is that if there is no table in the database yet, it will catch the error and just return the view.html with no data but for some reason it does not work anymore
Just realised I completely forgot to add the error, my bad
[2020-02-22 23:56:26,209] ERROR in app: Exception on /favicon.ico [GET]
Traceback (most recent call last):
File " -directory- ", line 316, in index
rows = cur.execute("select * from table")
sqlite3.OperationalError: no such table: table
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/anaconda3/lib/python3.7/site-packages/flask/app.py", line 2446, in wsgi_app
response = self.full_dispatch_request()
File "/opt/anaconda3/lib/python3.7/site-packages/flask/app.py", line 1951, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/opt/anaconda3/lib/python3.7/site-packages/flask/app.py", line 1820, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/opt/anaconda3/lib/python3.7/site-packages/flask/_compat.py", line 39, in reraise
raise value
File "/opt/anaconda3/lib/python3.7/site-packages/flask/app.py", line 1949, in full_dispatch_request
rv = self.dispatch_request()
File "/opt/anaconda3/lib/python3.7/site-packages/flask/app.py", line 1935, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File " -directory- /app/views.py", line 337, in index
content=render_template('pages/' + path))
File "/opt/anaconda3/lib/python3.7/site-packages/flask/templating.py", line 138, in render_template
ctx.app.jinja_env.get_or_select_template(template_name_or_list),
File "/opt/anaconda3/lib/python3.7/site-packages/jinja2/environment.py", line 869, in get_or_select_template
return self.get_template(template_name_or_list, parent, globals)
File "/opt/anaconda3/lib/python3.7/site-packages/jinja2/environment.py", line 830, in get_template
return self._load_template(name, self.make_globals(globals))
File "/opt/anaconda3/lib/python3.7/site-packages/jinja2/environment.py", line 804, in _load_template
template = self.loader.load(self, name, globals)
File "/opt/anaconda3/lib/python3.7/site-packages/jinja2/loaders.py", line 113, in load
source, filename, uptodate = self.get_source(environment, name)
File "/opt/anaconda3/lib/python3.7/site-packages/flask/templating.py", line 60, in get_source
return self._get_source_fast(environment, template)
File "/opt/anaconda3/lib/python3.7/site-packages/flask/templating.py", line 89, in _get_source_fast
raise TemplateNotFound(template)
jinja2.exceptions.TemplateNotFound: pages/favicon.ico
And just to note, the app runs fine on localhost, it just shows error now when it never did before
My Python server is raising random OSError: [errno 14] Bad Address when I use pynamodb Model. This issue is happening in any of my pynamodb Models and is very rare to occur. Most of the calls will be successful.
The error is raised when botocore reads from file /usr/local/lib/python3.7/site-packages/botocore/data/dynamodb/2012-08-10/service-2.json. I have no idea why this happens and why it is random.
I'm using Python 3.7.3, Django 2.2.2, Pynamodb 3.3.3 and Gunicorn 19.9.0. I'm running the server on Docker base python:3.7-slim managed by Kubernetes.
Example:
>>> from pynamodb.models import Model
>>> class User(Model):
... hash = UnicodeAttribute(hash_key=True)
... username = UnicodeAttribute(null=True)
... class Meta:
... table_name = "user"
... read_capacity_units = 10
... write_capacity_units = 10
... region = "us-west-2"
...
>>> user = User.get("valid-user-id")
Traceback (most recent call last):
File "<input>", line 11, in <module>
user = User.get("valid-user-id")
File "pynamodb/models.py", line 485, in get
hash_key, range_key = cls._serialize_keys(hash_key, range_key)
File "pynamodb/models.py", line 1372, in _serialize_keys
hash_key = cls._hash_key_attribute().serialize(hash_key)
File "pynamodb/models.py", line 1219, in _hash_key_attribute
hash_keyname = cls._get_meta_data().hash_keyname
File "pynamodb/models.py", line 1262, in _get_meta_data
cls._meta_table = MetaTable(cls._get_connection().describe_table())
File "pynamodb/connection/table.py", line 263, in describe_table
return self.connection.describe_table(self.table_name)
File "pynamodb/connection/base.py", line 659, in describe_table
tbl = self.get_meta_table(table_name, refresh=True)
File "pynamodb/connection/base.py", line 504, in get_meta_table
data = self.dispatch(DESCRIBE_TABLE, operation_kwargs)
File "pynamodb/connection/base.py", line 313, in dispatch
data = self._make_api_call(operation_name, operation_kwargs)
File "pynamodb/connection/base.py", line 341, in _make_api_call
operation_model = self.client._service_model.operation_model(operation_name)
File "pynamodb/connection/base.py", line 492, in client
self._client = self.session.create_client(SERVICE_NAME, self.region, endpoint_url=self.host)
File "botocore/session.py", line 838, in create_client
client_config=config, api_version=api_version)
File "botocore/client.py", line 79, in create_client
service_model = self._load_service_model(service_name, api_version)
File "botocore/client.py", line 117, in _load_service_model
api_version=api_version)
File "botocore/loaders.py", line 132, in _wrapper
data = func(self, *args, **kwargs)
File "botocore/loaders.py", line 383, in load_service_model
model = self.load_data(full_path)
File "botocore/loaders.py", line 132, in _wrapper
data = func(self, *args, **kwargs)
File "botocore/loaders.py", line 420, in load_data
found = self.file_loader.load_file(possible_path)
File "botocore/loaders.py", line 173, in load_file
payload = fp.read().decode('utf-8')
OSError: [Errno 14] Bad address
UPDATE:
This specific issue was due to a lack of memory in the pods. It happens when I integrate my server with Scout. Other than this issue the rest of the app worked fine, and the error didn't suggest a memory issue. It was fixed after I raise the memory in the pod.
I am trying to download a csv file from the internet. Here is my code using urllib. But I get HTTP Error 403.
Program-1:
from urllib import request
nse_stocks = 'https://www.nseindia.com/products/content/sec_bhavdata_full.csv'
def download_file(url):
connection = request.urlopen(url)
file_read = connection.read()
file_str = str(file_read)
lines_file_str = file_str.split('\\n')
file = open(r'downloaded_file.csv', 'w')
for line in lines_file_str:
file.write(line + '\n')
file.close()
download_file(nse_stocks)
Response:
Traceback (most recent call last):
File "C:/Users/sg0205481/Documents/Krishna/eBooks/Python/TheNewBoston/Python/downloadWebFile2.py", line 17, in <module>
download_file(nse_stocks)
File "C:/Users/sg0205481/Documents/Krishna/eBooks/Python/TheNewBoston/Python/downloadWebFile2.py", line 7, in download_file
connection = request.urlopen(url)
File "C:\Users\sg0205481\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\sg0205481\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 532, in open
response = meth(req, response)
File "C:\Users\sg0205481\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 642, in http_response
'http', request, response, code, msg, hdrs)
File "C:\Users\sg0205481\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 570, in error
return self._call_chain(*args)
File "C:\Users\sg0205481\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 504, in _call_chain
result = func(*args)
File "C:\Users\sg0205481\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 650, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 403: Forbidden
But I don't get the error with Program-2 with requests module. File gets downloaded successfully.
Program-2:
import requests
def download_file(url):
file_data = requests.get(url)
filename = 'downloaded_file.csv'
with open(filename, 'wb') as file:
file.write(file_data.content)
download_file('https://www.nseindia.com/products/content/sec_bhavdata_full.csv')
What is the problem with Program-1? What makes Program-2 pass successfully?
You can download a file (mime type) with get method from web server, not post method. If web server has handler for this extension and accept post method, than you may try!!!
I am using Newrelic's nginx-nr-agent but suddenly it crashed giving out the following error and not starting back.
2016-04-12 07:51:07,021 nginx-nr-agent [ERROR]: EXCEPTION: Traceback (most recent call last):
File "/usr/bin/nginx-nr-agent.py", line 431, in newrelic_push
u = urlopen(r, data=json.dumps(payload))
File "/usr/lib/python2.7/urllib2.py", line 127, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.7/urllib2.py", line 401, in open
response = self._open(req, data)
File "/usr/lib/python2.7/urllib2.py", line 419, in _open
'_open', req)
File "/usr/lib/python2.7/urllib2.py", line 379, in _call_chain
result = func(*args)
File "/usr/lib/python2.7/urllib2.py", line 1219, in https_open
return self.do_open(httplib.HTTPSConnection, req)
File "/usr/lib/python2.7/urllib2.py", line 1184, in do_open
r = h.getresponse(buffering=True)
File "/usr/lib/python2.7/httplib.py", line 1034, in getresponse
response.begin()
File "/usr/lib/python2.7/httplib.py", line 407, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.7/httplib.py", line 365, in _read_status
line = self.fp.readline()
File "/usr/lib/python2.7/socket.py", line 447, in readline
data = self._sock.recv(self._rbufsize)
File "/usr/lib/python2.7/ssl.py", line 241, in recv
return self.read(buflen)
File "/usr/lib/python2.7/ssl.py", line 160, in read
return self._sslobj.read(len)
File "/usr/lib/pymodules/python2.7/daemon/daemon.py", line 399, in terminate
raise exception
SystemExit: Terminating on signal 15
Any help please?
I got two solution for this from here, one is to remove the pid file /var/run/nginx-nr-agent/nginx-nr-agent.pid and than start/restart it. And second solution is to restart the server.
This plugin is published and supported by Nginx, Inc. If the previous suggestions did not help, contact
newrelic-support#nginx.com.