I want to create a web app on GAE in Go and I need it to listen to a custom port. How do I set it up? I tried to use http.ListenAndServe(":12345", nil) in the sandbox, but got this in my console:
WARNING 2011-10-07 20:01:01,252 urlfetch_stub.py:108] No ssl package found. urlfetch
will not be able to validate SSL certificates.
INFO 2011-10-07 20:01:01,847 appengine_rpc.py:159] Server: appengine.google.com
INFO 2011-10-07 20:01:01,855 appcfg.py:463] Checking for updates to the SDK.
INFO 2011-10-07 20:01:04,625 appcfg.py:480] The SDK is up to date.
WARNING 2011-10-07 20:01:04,625 datastore_file_stub.py:512] Could not read datastore data from /var/folders/e0/e0U5NtMXFHOB6jVwkVhsnE+++TM/-Tmp-/dev_appserver.datastore
INFO 2011-10-07 20:01:04,627 rdbms_sqlite.py:58] Connecting to SQLite database '' with file '/var/folders/e0/e0U5NtMXFHOB6jVwkVhsnE+++TM/-Tmp-/dev_appserver.rdbms'
WARNING 2011-10-07 20:01:04,633 dev_appserver.py:4748] Could not initialize images API; you are likely missing the Python "PIL" module. ImportError: No module named _imaging
INFO 2011-10-07 20:01:04,641 dev_appserver_multiprocess.py:637] Running application tpbitblock on port 8080: http://localhost:8080
INFO 2011-10-07 20:01:08,448 __init__.py:365] building _go_app
INFO 2011-10-07 20:01:11,105 __init__.py:351] running _go_app
throw: init rescheduling
runtime.throw+0x40 /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/runtime.c:102
runtime.throw(0x278c7f, 0xdc73)
schedule+0x3e /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/proc.c:535
schedule(0x10803000, 0x10803000)
runtime.mcall+0x3a /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/386/asm.s:174
runtime.mcall(0x10803000, 0x0)
goroutine 2 [1]:
net.*pollServer·Run /private/tmp/appengine/google_appengine/goroot/src/pkg/net/fd.go:213
net.*pollServer·Run(0x10801c90, 0x0)
runtime.goexit /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/proc.c:178
runtime.goexit()
----- goroutine created by -----
net.newPollServer+0x2dd /private/tmp/appengine/google_appengine/goroot/src/pkg/net/newpollserver.go:39
goroutine 1 [4]:
runtime.gosched+0x4d /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/proc.c:603
runtime.gosched()
runtime.chanrecv+0x14d /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/chan.c:361
runtime.chanrecv(0x1081d900, 0x30800e6c, 0x0, 0x0, 0x0, ...)
runtime.chanrecv1+0x3f /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/chan.c:424
runtime.chanrecv1(0x1081d900, 0x1081e580)
net.*pollServer·WaitRead+0x47 /private/tmp/appengine/google_appengine/goroot/src/pkg/net/fd.go:255
net.*pollServer·WaitRead(0x10801c90, 0x1081e580, 0x0)
net.*netFD·accept+0x2ae /private/tmp/appengine/google_appengine/goroot/src/pkg/net/fd.go:605
net.*netFD·accept(0x1081e580, 0x45716, 0x0, 0x0, 0x0, ...)
net.*TCPListener·AcceptTCP+0x53 /private/tmp/appengine/google_appengine/goroot/src/pkg/net/tcpsock.go:272
net.*TCPListener·AcceptTCP(0x108008b8, 0x30800f00, 0x0, 0x0)
net.*TCPListener·Accept+0x36 /private/tmp/appengine/google_appengine/goroot/src/pkg/net/tcpsock.go:282
net.*TCPListener·Accept(0x108008b8, 0x0, 0x0, 0x0, 0x0, ...)
http.*Server·Serve+0x96 /private/tmp/appengine/google_appengine/goroot/src/pkg/http/server.go:868
http.*Server·Serve(0x10839d20, 0x10839e20, 0x108008b8, 0x0, 0x0, ...)
http.*Server·ListenAndServe+0x9c /private/tmp/appengine/google_appengine/goroot/src/pkg/http/server.go:855
http.*Server·ListenAndServe(0x10839d20, 0x10839d20, 0x406)
http.ListenAndServe+0x52 /private/tmp/appengine/google_appengine/goroot/src/pkg/http/server.go:920
http.ListenAndServe(0x155640, 0x6, 0x0, 0x0, 0x0, ...)
BitBlock.init·1+0x174 BitBlock/BitBlock.go:43
BitBlock.init·1()
BitBlock.init+0x70 BitBlock/BitBlock.go:0
BitBlock.init()
main.init+0x39 _go_main.go:0
main.init()
runtime.mainstart+0x5 /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/386/asm.s:91
runtime.mainstart()
runtime.goexit /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/proc.c:178
runtime.goexit()
----- goroutine created by -----
_rt0_386+0xbf /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/386/asm.s:80
ERROR 2011-10-07 20:01:12,196 dev_appserver.py:4200] Exception encountered handling request
Traceback (most recent call last):
File "/Google App Engine/google/appengine/tools/dev_appserver.py", line 4143, in _HandleRequest
self._Dispatch(dispatcher, self.rfile, outfile, env_dict)
File "/Google App Engine/google/appengine/tools/dev_appserver.py", line 4049, in _Dispatch
base_env_dict=env_dict)
File "/Google App Engine/google/appengine/tools/dev_appserver.py", line 616, in Dispatch
base_env_dict=base_env_dict)
File "/Google App Engine/google/appengine/tools/dev_appserver.py", line 3120, in Dispatch
self._module_dict)
File "/Google App Engine/google/appengine/tools/dev_appserver.py", line 2972, in ExecuteCGI
env, infile, outfile)
File "/Google App Engine/google/appengine/ext/go/__init__.py", line 399, in execute_go_cgi
GO_APP.make_and_run()
File "/Google App Engine/google/appengine/ext/go/__init__.py", line 362, in make_and_run
wait_until_go_app_ready(self.proc.pid)
File "/Google App Engine/google/appengine/ext/go/__init__.py", line 285, in wait_until_go_app_ready
raise Exception('unable to start ' + GO_APP_NAME)
Exception: unable to start _go_app
INFO 2011-10-07 20:01:12,242 dev_appserver.py:4247] "GET / HTTP/1.1" 500 -
INFO 2011-10-07 20:01:12,505 __init__.py:351] running _go_app
throw: init rescheduling
runtime.throw+0x40 /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/runtime.c:102
runtime.throw(0x278c7f, 0xdc73)
schedule+0x3e /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/proc.c:535
schedule(0x10803000, 0x10803000)
runtime.mcall+0x3a /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/386/asm.s:174
runtime.mcall(0x10803000, 0x0)
goroutine 2 [1]:
net.*pollServer·Run /private/tmp/appengine/google_appengine/goroot/src/pkg/net/fd.go:213
net.*pollServer·Run(0x10801c90, 0x0)
runtime.goexit /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/proc.c:178
runtime.goexit()
----- goroutine created by -----
net.newPollServer+0x2dd /private/tmp/appengine/google_appengine/goroot/src/pkg/net/newpollserver.go:39
goroutine 1 [4]:
runtime.gosched+0x4d /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/proc.c:603
runtime.gosched()
runtime.chanrecv+0x14d /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/chan.c:361
runtime.chanrecv(0x1081d900, 0x30800e6c, 0x0, 0x0, 0x0, ...)
runtime.chanrecv1+0x3f /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/chan.c:424
runtime.chanrecv1(0x1081d900, 0x1081e580)
net.*pollServer·WaitRead+0x47 /private/tmp/appengine/google_appengine/goroot/src/pkg/net/fd.go:255
net.*pollServer·WaitRead(0x10801c90, 0x1081e580, 0x0)
net.*netFD·accept+0x2ae /private/tmp/appengine/google_appengine/goroot/src/pkg/net/fd.go:605
net.*netFD·accept(0x1081e580, 0x45716, 0x0, 0x0, 0x0, ...)
net.*TCPListener·AcceptTCP+0x53 /private/tmp/appengine/google_appengine/goroot/src/pkg/net/tcpsock.go:272
net.*TCPListener·AcceptTCP(0x108008b8, 0x30800f00, 0x0, 0x0)
net.*TCPListener·Accept+0x36 /private/tmp/appengine/google_appengine/goroot/src/pkg/net/tcpsock.go:282
net.*TCPListener·Accept(0x108008b8, 0x0, 0x0, 0x0, 0x0, ...)
http.*Server·Serve+0x96 /private/tmp/appengine/google_appengine/goroot/src/pkg/http/server.go:868
http.*Server·Serve(0x10839d20, 0x10839e20, 0x108008b8, 0x0, 0x0, ...)
http.*Server·ListenAndServe+0x9c /private/tmp/appengine/google_appengine/goroot/src/pkg/http/server.go:855
http.*Server·ListenAndServe(0x10839d20, 0x10839d20, 0x406)
http.ListenAndServe+0x52 /private/tmp/appengine/google_appengine/goroot/src/pkg/http/server.go:920
http.ListenAndServe(0x155640, 0x6, 0x0, 0x0, 0x0, ...)
BitBlock.init·1+0x174 BitBlock/BitBlock.go:43
BitBlock.init·1()
BitBlock.init+0x70 BitBlock/BitBlock.go:0
BitBlock.init()
main.init+0x39 _go_main.go:0
main.init()
runtime.mainstart+0x5 /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/386/asm.s:91
runtime.mainstart()
runtime.goexit /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/proc.c:178
runtime.goexit()
----- goroutine created by -----
_rt0_386+0xbf /private/tmp/appengine/google_appengine/goroot/src/pkg/runtime/386/asm.s:80
ERROR 2011-10-07 20:01:13,530 dev_appserver.py:4200] Exception encountered handling request
Traceback (most recent call last):
File "/Google App Engine/google/appengine/tools/dev_appserver.py", line 4143, in _HandleRequest
self._Dispatch(dispatcher, self.rfile, outfile, env_dict)
File "/Google App Engine/google/appengine/tools/dev_appserver.py", line 4049, in _Dispatch
base_env_dict=env_dict)
File "/Google App Engine/google/appengine/tools/dev_appserver.py", line 616, in Dispatch
base_env_dict=base_env_dict)
File "/Google App Engine/google/appengine/tools/dev_appserver.py", line 3120, in Dispatch
self._module_dict)
File "/Google App Engine/google/appengine/tools/dev_appserver.py", line 2972, in ExecuteCGI
env, infile, outfile)
File "/Google App Engine/google/appengine/ext/go/__init__.py", line 399, in execute_go_cgi
GO_APP.make_and_run()
File "/Google App Engine/google/appengine/ext/go/__init__.py", line 362, in make_and_run
wait_until_go_app_ready(self.proc.pid)
File "/Google App Engine/google/appengine/ext/go/__init__.py", line 285, in wait_until_go_app_ready
raise Exception('unable to start ' + GO_APP_NAME)
Exception: unable to start _go_app
INFO 2011-10-07 20:01:13,531 dev_appserver.py:4247] "GET /favicon.ico HTTP/1.1" 500 -
You can't do that. App Engine doesn't allow custom http ports. App Engine environment is heavily sandboxed and not everything that you can do in pure Go (or Java or Python) is available in their respective App Engine environments.
From http://code.google.com/appengine/docs/go/overview.html:
As with the Java and Python environments, not all the standard library's functionality is available inside the sandbox. For example, attempts to open a socket or write to a file will return an os.EINVAL error.
Go apps run inside a secure "sandbox" environment with a reduced set of libraries. For instance, an app cannot write data to the local file system or make arbitrary network connections. Instead, apps use scalable services provided by App Engine to store data and communicate over the Internet.
It is indeed possible, all you need to do is goapp serve --port=3000 (or whatever port you'd like).
If you're trying to access your goapp on a VM (parallels), like I was, you can do goapp serve --host=0.0.0.0
Documentation: https://cloud.google.com/appengine/docs/go/tools/devserver#Go_Command-line_arguments
Related
We deployed Apache Airflow 2.3.3 to Azure.
Webserver - Web App
Scheduler - ACI
Celery Worker - ACI
We were seeing errors on the Celery ACI console related to Postgres and Redis connection timeouts as below
[2022-09-22 18:55:50,650: WARNING/ForkPoolWorker-15] Failed operation _store_result. Retrying 2 more times.
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1803, in _execute_context
cursor, statement, parameters, context
File "/home/airflow/.local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 719, in do_execute
cursor.execute(statement, parameters)
psycopg2.DatabaseError: could not receive data from server: Connection timed out
SSL SYSCALL error: Connection timed out
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.7/site-packages/celery/backends/database/__init__.py", line 47, in _inner
return fun(*args, **kwargs)
File "/home/airflow/.local/lib/python3.7/site-packages/celery/backends/database/__init__.py", line 117, in _store_result
task = list(session.query(self.task_cls).filter(self.task_cls.task_id == task_id))
File "/home/airflow/.local/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 2887, in __iter__
return self._iter().__iter__()
File "/home/airflow/.local/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 2897, in _iter
execution_options={"_sa_orm_load_options": self.load_options},
File "/home/airflow/.local/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 1689, in execute
result = conn._execute_20(statement, params or {}, execution_options)
File "/home/airflow/.local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1614, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/home/airflow/.local/lib/python3.7/site-packages/sqlalchemy/sql/elements.py", line 326, in _execute_on_connection
self, multiparams, params, execution_options
File "/home/airflow/.local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1491, in _execute_clauseelement
cache_hit=cache_hit,
File "/home/airflow/.local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1846, in _execute_context
e, statement, parameters, cursor, context
File "/home/airflow/.local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 2027, in _handle_dbapi_exception
sqlalchemy_exception, with_traceback=exc_info[2], from_=e
File "/home/airflow/.local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/home/airflow/.local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1803, in _execute_context
cursor, statement, parameters, context
File "/home/airflow/.local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 719, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.DatabaseError: (psycopg2.DatabaseError) could not receive data from server: Connection timed out
SSL SYSCALL error: Connection timed out
[SQL: SELECT celery_taskmeta.id AS celery_taskmeta_id, celery_taskmeta.task_id AS celery_taskmeta_task_id, celery_taskmeta.status AS celery_taskmeta_status, celery_taskmeta.result AS celery_taskmeta_result, celery_taskmeta.date_done AS celery_taskmeta_date_done, celery_taskmeta.traceback AS celery_taskmeta_traceback
FROM celery_taskmeta
WHERE celery_taskmeta.task_id = %(task_id_1)s]
[parameters: {'task_id_1': 'c5f9f53c-8afe-4d67-8d3b-d7ad84875de1'}]
(Background on this error at: https://sqlalche.me/e/14/4xp6)
[2022-09-22 18:55:50,929: INFO/ForkPoolWorker-15] [c5f9f53c-8afe-4d67-8d3b-d7ad84875de1] Executing command in Celery: ['airflow', 'tasks', 'run', 'CS_ALERTING', 'CheckRunningTasks', 'scheduled__2022-09-22T18:00:00+00:00', '--local', '--subdir', 'DAGS_FOLDER/CS_ALERTING.py']
[2022-09-22 18:55:51,241: INFO/ForkPoolWorker-15] Filling up the DagBag from /opt/airflow/platform_pam/dags/CS_ALERTING.py
[2022-09-22 18:55:53,467: INFO/ForkPoolWorker-15] Running <TaskInstance: CS_ALERTING.CheckRunningTasks scheduled__2022-09-22T18:00:00+00:00 [queued]> on host localhost
[2022-09-22 18:55:58,304: INFO/ForkPoolWorker-15] Task airflow.executors.celery_executor.execute_command[c5f9f53c-8afe-4d67-8d3b-d7ad84875de1] succeeded in 960.1964174450004s: None
[2022-09-22 19:29:25,931: WARNING/MainProcess] consumer: Connection to broker lost. Trying to re-establish the connection...
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.7/site-packages/redis/connection.py", line 706, in send_packed_command
sendall(self._sock, item)
File "/home/airflow/.local/lib/python3.7/site-packages/redis/_compat.py", line 9, in sendall
return sock.sendall(*args, **kwargs)
File "/usr/local/lib/python3.7/ssl.py", line 1034, in sendall
v = self.send(byte_view[count:])
File "/usr/local/lib/python3.7/ssl.py", line 1003, in send
return self._sslobj.write(data)
TimeoutError: [Errno 110] Connection timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.7/site-packages/celery/worker/consumer/consumer.py", line 332, in start
blueprint.start(self)
File "/home/airflow/.local/lib/python3.7/site-packages/celery/bootsteps.py", line 116, in start
step.start(parent)
File "/home/airflow/.local/lib/python3.7/site-packages/celery/worker/consumer/consumer.py", line 628, in start
c.loop(*c.loop_args())
File "/home/airflow/.local/lib/python3.7/site-packages/celery/worker/loops.py", line 97, in asynloop
next(loop)
File "/home/airflow/.local/lib/python3.7/site-packages/kombu/asynchronous/hub.py", line 301, in create_loop
poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
File "/home/airflow/.local/lib/python3.7/site-packages/kombu/asynchronous/hub.py", line 143, in fire_timers
entry()
File "/home/airflow/.local/lib/python3.7/site-packages/kombu/asynchronous/timer.py", line 64, in __call__
return self.fun(*self.args, **self.kwargs)
File "/home/airflow/.local/lib/python3.7/site-packages/kombu/asynchronous/timer.py", line 126, in _reschedules
return fun(*args, **kwargs)
File "/home/airflow/.local/lib/python3.7/site-packages/kombu/transport/redis.py", line 557, in maybe_check_subclient_health
client.check_health()
File "/home/airflow/.local/lib/python3.7/site-packages/redis/client.py", line 3522, in check_health
check_health=False)
File "/home/airflow/.local/lib/python3.7/site-packages/redis/connection.py", line 726, in send_command
check_health=kwargs.get('check_health', True))
File "/home/airflow/.local/lib/python3.7/site-packages/redis/connection.py", line 718, in send_packed_command
(errno, errmsg))
redis.exceptions.ConnectionError: Error 110 while writing to socket. Connection timed out.
[2022-09-22 19:29:26,023: WARNING/MainProcess] /home/airflow/.local/lib/python3.7/site-packages/celery/worker/consumer/consumer.py:367: CPendingDeprecationWarning:
I referred the Airflow's documentation and found setting up database
We are modifying Airflow's docker image and adding a python file,
airflow.www.db_utils.db_config (This file is installed to site_packages) and defined the dictionary
keepalive_kwargs = {
"keepalives": 1,
"keepalives_idle": 30,
"keepalives_interval": 5,
"keepalives_count": 5,
}
Finally, we are setting
ENV AIRFLOW__DATABASE__SQL_ALCHEMY_CONNECT_ARGS="airflow.www.db_utils.db_config.keepalive_kwargs"
Unfortunately, the error stills persist. It will be great if someone helps me to resolve this issue.
my ~/.aws/credentials looks like
[default]
aws_access_key_id = XYZ
aws_secret_access_key = ABC
[testing]
source_profile = default
role_arn = arn:aws:iam::54:role/ad
I add my remote like
dvc remote add --local -v myremote s3://bib-ds-models-testing/data/dvc-test
I have made my .dvc/config.local to look like
[‘remote “myremote”’]
url = s3://bib-ds-models-testing/data/dvc-test
access_key_id = XYZ
secret_access_key = ABC/h2hOsRcCIFqwYWV7eZaUq3gNmS
profile=‘testing’
credentialpath = /Users/nyt21/.aws/credentials
but still after running dvc push -r myremote I get
ERROR: unexpected error - Forbidden: An error occurred (403) when calling the HeadObject operation: Forbidden
** Update
here is the output of dvc push -v
2021-07-25 22:40:38,887 DEBUG: Check for update is enabled.
2021-07-25 22:40:39,022 DEBUG: Preparing to upload data to 's3://bib-ds-models-testing/data/dvc-test'
2021-07-25 22:40:39,022 DEBUG: Preparing to collect status from s3://bib-ds-models-testing/data/dvc-test
2021-07-25 22:40:39,022 DEBUG: Collecting information from local cache...
2021-07-25 22:40:39,022 DEBUG: Collecting information from remote cache...
2021-07-25 22:40:39,022 DEBUG: Matched '0' indexed hashes
2021-07-25 22:40:39,022 DEBUG: Querying 1 hashes via object_exists
2021-07-25 22:40:39,644 ERROR: unexpected error - Forbidden: An error occurred (403) when calling the HeadObject operation: Forbidden
------------------------------------------------------------
Traceback (most recent call last):
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/s3fs/core.py", line 246, in _call_s3
out = await method(**additional_kwargs)
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/aiobotocore/client.py", line 154, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the ListObjectsV2 operation: Access Denied
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/s3fs/core.py", line 1057, in _info
out = await self._simple_info(path)
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/s3fs/core.py", line 970, in _simple_info
out = await self._call_s3(
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/s3fs/core.py", line 265, in _call_s3
raise translate_boto_error(err)
PermissionError: Access Denied
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/s3fs/core.py", line 246, in _call_s3
out = await method(**additional_kwargs)
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/aiobotocore/client.py", line 154, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (403) when calling the HeadObject operation: Forbidden
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/main.py", line 55, in main
ret = cmd.do_run()
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/command/base.py", line 50, in do_run
return self.run()
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/command/data_sync.py", line 57, in run
processed_files_count = self.repo.push(
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/repo/__init__.py", line 51, in wrapper
return f(repo, *args, **kwargs)
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/repo/push.py", line 44, in push
pushed += self.cloud.push(objs, jobs, remote=remote)
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/data_cloud.py", line 79, in push
return remote_obj.push(
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/remote/base.py", line 57, in wrapper
return f(obj, *args, **kwargs)
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/remote/base.py", line 494, in push
ret = self._process(
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/remote/base.py", line 351, in _process
dir_status, file_status, dir_contents = self._status(
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/remote/base.py", line 195, in _status
self.hashes_exist(
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/remote/base.py", line 145, in hashes_exist
return indexed_hashes + self.odb.hashes_exist(list(hashes), **kwargs)
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/objects/db/base.py", line 438, in hashes_exist
remote_hashes = self.list_hashes_exists(hashes, jobs, name)
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/objects/db/base.py", line 389, in list_hashes_exists
ret = list(itertools.compress(hashes, in_remote))
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
yield fs.pop().result()
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/concurrent/futures/_base.py", line 444, in result
return self.__get_result()
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
raise self._exception
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/objects/db/base.py", line 380, in exists_with_progress
ret = self.fs.exists(path_info)
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/dvc/fs/fsspec_wrapper.py", line 92, in exists
return self.fs.exists(self._with_bucket(path_info))
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/fsspec/asyn.py", line 87, in wrapper
return sync(self.loop, func, *args, **kwargs)
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/fsspec/asyn.py", line 68, in sync
raise result[0]
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/fsspec/asyn.py", line 24, in _runner
result[0] = await coro
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/s3fs/core.py", line 802, in _exists
await self._info(path, bucket, key, version_id=version_id)
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/s3fs/core.py", line 1061, in _info
out = await self._version_aware_info(path, version_id)
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/s3fs/core.py", line 1004, in _version_aware_info
out = await self._call_s3(
File "/Users/nyt21/opt/miniconda3/envs/dvc/lib/python3.8/site-packages/s3fs/core.py", line 265, in _call_s3
raise translate_boto_error(err)
PermissionError: Forbidden
------------------------------------------------------------
2021-07-25 22:40:39,712 DEBUG: Version info for developers:
DVC version: 2.5.4 (pip)
---------------------------------
Platform: Python 3.8.10 on macOS-10.16-x86_64-i386-64bit
Supports:
http (requests = 2.26.0),
https (requests = 2.26.0),
s3 (s3fs = 2021.6.1, boto3 = 1.18.6)
Cache types: reflink, hardlink, symlink
Cache directory: apfs on /dev/disk3s1s1
Caches: local
Remotes: s3
Workspace directory: apfs on /dev/disk3s1s1
Repo: dvc, git
Having any troubles? Hit us up at https://dvc.org/support, we are always happy to help!
2021-07-25 22:40:39,713 DEBUG: Analytics is enabled.
2021-07-25 22:40:39,765 DEBUG: Trying to spawn '['daemon', '-q', 'analytics', '/var/folders/4x/xhm22wt16gl6m9nvkl9gllkc0000gn/T/tmpo86jdns5']'
2021-07-25 22:40:39,769 DEBUG: Spawned '['daemon', '-q', 'analytics', '/var/folders/4x/xhm22wt16gl6m9nvkl9gllkc0000gn/T/tmpo86jdns5']'
I can upload through python
import boto3
import os
import pickle
bucket_name = 'bib-ds-models-testing'
os.environ["AWS_PROFILE"] = "testing"
session = boto3.Session()
s3_client = boto3.client('s3')
s3_client.upload_file('/Users/nyt21/Devel/DVC/test/data/iris.csv',
'bib-ds-models-testing',
'data/dvc-test/my_iris.csv')
I don't use aws CLI but the following also gives an access deny !
aws s3 ls s3://bib-ds-models-testing/data/dvc-test
An error occurred (AccessDenied) when calling the ListObjectsV2
operation: Access Denied
but it works if I add --profile=testing
aws s3 ls s3://bib-ds-models-testing/data/dvc-test --profile=testing
PRE dvc-test/
just you know environment variable AWS_PROFILE is already set to 'testing'
UPDATE
I have tried both AWS_PROFILE='testing' and AWS_PROFILE=testing, neither of them worked.
I'm using R blogdown blogdown:::serve_site(),and the result is below:
Error in shell(cmd, mustWork = TRUE, intern = intern) :
'"C:\Users\xsong\AppData\Roaming\Hugo\hugo.exe" new site "." -f toml' execution failed with error code 1
In addition: Warning message:
In blogdown::new_site() : The directory '.' is not empty
> blogdown::new_site()
Congratulations! Your new Hugo site is created in F:\wwe.
Just a few more steps and you're ready to go:
1. Download a theme into the same-named folder.
Choose a theme from https://themes.gohugo.io/, or
create your own with the "hugo new theme <THEMENAME>" command.
2. Perhaps you want to add some content. You can add single files
with "hugo new <SECTIONNAME>\<FILENAME>.<FORMAT>".
3. Start the built-in live server via "hugo server".
Visit https://gohugo.io/ for quickstart guide and full documentation.
trying URL 'https://github.com/yihui/hugo-lithium/archive/master.zip'
downloaded 117 KB
processing 7 content files
Building sites 鈥?panic: Too many homes
goroutine 76 [running]:
github.com/gohugoio/hugo/hugolib.(*HugoSites).createMissingPages(0xc00030f180, 0x0, 0x0)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites.go:627 +0x111f
github.com/gohugoio/hugo/hugolib.(*HugoSites).assemble(0xc00030f180, 0xc000389c00, 0xc0005cdc70, 0x20)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:245 +0x141
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build.func2.2()
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:98 +0x43
runtime/trace.WithRegion(0x14f5060, 0xc0004d4ba0, 0x129dfd2, 0x8, 0xc0005cdd20)
/usr/local/go/src/runtime/trace/annotation.go:137 +0xf0
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build.func2(0xc00049bdc0, 0x20)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:100 +0x244
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build.func3()
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:109 +0x36
runtime/trace.WithRegion(0x14f5060, 0xc0004d4ba0, 0x129b050, 0x7, 0xc00049bec8)
/usr/local/go/src/runtime/trace/annotation.go:137 +0xf0
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build(0xc00030f180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:111 +0x7c4
github.com/gohugoio/hugo/commands.(*commandeer).buildSites(...)
/go/src/github.com/gohugoio/hugo/commands/hugo.go:763
github.com/gohugoio/hugo/commands.(*commandeer).fullBuild.func3(0x8, 0x134c5d8)
/go/src/github.com/gohugoio/hugo/commands/hugo.go:313 +0x80
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0004d4b70, 0xc0002f0d30)
/go/pkg/mod/golang.org/x/sync#v0.0.0-20180314180146-1d60e4601c6f/errgroup/errgroup.go:58 +0x5e
created by golang.org/x/sync/errgroup.(*Group).Go
/go/pkg/mod/golang.org/x/sync#v0.0.0-20180314180146-1d60e4601c6f/errgroup/errgroup.go:55 +0x6d
Building sites 鈥?panic: Too many homes
goroutine 85 [running]:
github.com/gohugoio/hugo/hugolib.(*HugoSites).createMissingPages(0xc000435180, 0x0, 0x0)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites.go:627 +0x111f
github.com/gohugoio/hugo/hugolib.(*HugoSites).assemble(0xc000435180, 0xc0003bce00, 0xc0005e1c70, 0x20)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:245 +0x141
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build.func2.2()
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:98 +0x43
runtime/trace.WithRegion(0x14f5060, 0xc0003c13e0, 0x129dfd2, 0x8, 0xc0005e1d20)
/usr/local/go/src/runtime/trace/annotation.go:137 +0xf0
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build.func2(0xc000355dc0, 0x20)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:100 +0x244
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build.func3()
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:109 +0x36
runtime/trace.WithRegion(0x14f5060, 0xc0003c13e0, 0x129b050, 0x7, 0xc000355ec8)
/usr/local/go/src/runtime/trace/annotation.go:137 +0xf0
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build(0xc000435180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:111 +0x7c4
github.com/gohugoio/hugo/commands.(*commandeer).buildSites(...)
/go/src/github.com/gohugoio/hugo/commands/hugo.go:763
github.com/gohugoio/hugo/commands.(*commandeer).fullBuild.func3(0x8, 0x134c5d8)
/go/src/github.com/gohugoio/hugo/commands/hugo.go:313 +0x80
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0003c13b0, 0xc000215ae0)
/go/pkg/mod/golang.org/x/sync#v0.0.0-20180314180146-1d60e4601c6f/errgroup/errgroup.go:58 +0x5e
created by golang.org/x/sync/errgroup.(*Group).Go
/go/pkg/mod/golang.org/x/sync#v0.0.0-20180314180146-1d60e4601c6f/errgroup/errgroup.go:55 +0x6d
系统找不到指定的路径。
Error in shell(cmd, mustWork = TRUE, intern = intern) :
'"C:\Users\xsong\AppData\Roaming\Hugo\hugo.exe" -b / -D -F -d "public" --themesDir themes -t hugo-lithium' execution failed with error code 1
> blogdown:::serve_site()
Building sites 鈥?panic: Too many homes
goroutine 73 [running]:
github.com/gohugoio/hugo/hugolib.(*HugoSites).createMissingPages(0xc0002e12d0, 0x0, 0x0)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites.go:627 +0x111f
github.com/gohugoio/hugo/hugolib.(*HugoSites).assemble(0xc0002e12d0, 0xc000472000, 0xc000549c70, 0x20)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:245 +0x141
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build.func2.2()
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:98 +0x43
runtime/trace.WithRegion(0x14f5060, 0xc0001b2120, 0x129dfd2, 0x8, 0xc000549d20)
/usr/local/go/src/runtime/trace/annotation.go:137 +0xf0
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build.func2(0xc0005afdc0, 0x20)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:100 +0x244
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build.func3()
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:109 +0x36
runtime/trace.WithRegion(0x14f5060, 0xc0001b2120, 0x129b050, 0x7, 0xc0005afec8)
/usr/local/go/src/runtime/trace/annotation.go:137 +0xf0
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build(0xc0002e12d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:111 +0x7c4
github.com/gohugoio/hugo/commands.(*commandeer).buildSites(...)
/go/src/github.com/gohugoio/hugo/commands/hugo.go:763
github.com/gohugoio/hugo/commands.(*commandeer).fullBuild.func3(0x8, 0x134c5d8)
/go/src/github.com/gohugoio/hugo/commands/hugo.go:313 +0x80
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0001b20f0, 0xc0005800e0)
/go/pkg/mod/golang.org/x/sync#v0.0.0-20180314180146-1d60e4601c6f/errgroup/errgroup.go:58 +0x5e
created by golang.org/x/sync/errgroup.(*Group).Go
/go/pkg/mod/golang.org/x/sync#v0.0.0-20180314180146-1d60e4601c6f/errgroup/errgroup.go:55 +0x6d
Building sites 鈥?panic: Too many homes
goroutine 31 [running]:
github.com/gohugoio/hugo/hugolib.(*HugoSites).createMissingPages(0xc000097180, 0x0, 0x0)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites.go:627 +0x111f
github.com/gohugoio/hugo/hugolib.(*HugoSites).assemble(0xc000097180, 0xc000298000, 0xc0005efc70, 0x20)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:245 +0x141
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build.func2.2()
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:98 +0x43
runtime/trace.WithRegion(0x14f5060, 0xc00040c4e0, 0x129dfd2, 0x8, 0xc0005efd20)
/usr/local/go/src/runtime/trace/annotation.go:137 +0xf0
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build.func2(0xc0002b9dc0, 0x20)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:100 +0x244
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build.func3()
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:109 +0x36
runtime/trace.WithRegion(0x14f5060, 0xc00040c4e0, 0x129b050, 0x7, 0xc0002b9ec8)
/usr/local/go/src/runtime/trace/annotation.go:137 +0xf0
github.com/gohugoio/hugo/hugolib.(*HugoSites).Build(0xc000097180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/gohugoio/hugo/hugolib/hugo_sites_build.go:111 +0x7c4
github.com/gohugoio/hugo/commands.(*commandeer).buildSites(...)
/go/src/github.com/gohugoio/hugo/commands/hugo.go:763
github.com/gohugoio/hugo/commands.(*commandeer).fullBuild.func3(0x8, 0x134c5d8)
/go/src/github.com/gohugoio/hugo/commands/hugo.go:313 +0x80
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc00040c390, 0xc000112030)
/go/pkg/mod/golang.org/x/sync#v0.0.0-20180314180146-1d60e4601c6f/errgroup/errgroup.go:58 +0x5e
created by golang.org/x/sync/errgroup.(*Group).Go
/go/pkg/mod/golang.org/x/sync#v0.0.0-20180314180146-1d60e4601c6f/errgroup/errgroup.go:55 +0x6d
系统找不到指定的路径。
Error in shell(cmd, mustWork = TRUE, intern = intern) :
'"C:\Users\xsong\AppData\Roaming\Hugo\hugo.exe" -b / -D -F -d "public" --themesDir themes -t hugo-lithium' execution failed with error code 1
It seems that there are some problems of hugo lithium theme. I also cannot successfully use blogdown::new_site() to bulid a new site. I have tried to reinstall blogdown and hugo, but it does not work.
Please help me.
I've been working on an API for authentication, and while trying to deploy it on a server I stumbled upon this odd bug. The code works perfectly fine on my laptop, but as soon as I try to deploy it, this error occurs:
PANIC: runtime error: invalid memory address or nil pointer dereference
goroutine 21 [running]:
github.com/urfave/negroni.(*Recovery).ServeHTTP.func1(0x7f5771b811e8, 0xc4200980e8, 0xc42009a870, 0xc420138800)
/home/linux/go/src/github.com/urfave/negroni/recovery.go:159 +0xef
panic(0x84ad60, 0xcc4f30)
/usr/local/go/src/runtime/panic.go:502 +0x229
database/sql.(*Rows).close(0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/database/sql/sql.go:2907 +0x6b
database/sql.(*Rows).Close(0x0, 0x0, 0x0)
/usr/local/go/src/database/sql/sql.go:2903 +0x33
panic(0x8408e0, 0xc42008ab70)
/usr/local/go/src/runtime/panic.go:502 +0x229
github.com/lars250698/iotmap/api/models.PassQuery(0xc420096d38, 0x4, 0xc4201129a0, 0x0)
/home/linux/go/src/github.com/lars250698/iotmap/api/models/user.go:35 +0x236
github.com/lars250698/iotmap/api/controllers.Auth(0x7f5771b811e8, 0xc4200980e8, 0xc420138b00)
/home/linux/go/src/github.com/lars250698/iotmap/api/controllers/auth.go:28 +0x151
net/http.HandlerFunc.ServeHTTP(0x8d1d60, 0x7f5771b811e8, 0xc4200980e8, 0xc420138b00)
/usr/local/go/src/net/http/server.go:1947 +0x44
github.com/gorilla/mux.(*Router).ServeHTTP(0xc42013e850, 0x7f5771b811e8, 0xc4200980e8, 0xc420138b00)
/home/linux/go/src/github.com/gorilla/mux/mux.go:162 +0xed
github.com/gorilla/handlers.(*cors).ServeHTTP(0xc42014a480, 0x7f5771b811e8, 0xc4200980e8, 0xc420138800)
/home/linux/go/src/github.com/gorilla/handlers/cors.go:52 +0xa3b
github.com/urfave/negroni.Wrap.func1(0x7f5771b811e8, 0xc4200980e8, 0xc420138800, 0xc420112980)
/home/linux/go/src/github.com/urfave/negroni/negroni.go:46 +0x4d
github.com/urfave/negroni.HandlerFunc.ServeHTTP(0xc420112800, 0x7f5771b811e8, 0xc4200980e8, 0xc420138800, 0xc420112980)
/home/linux/go/src/github.com/urfave/negroni/negroni.go:29 +0x4e
github.com/urfave/negroni.middleware.ServeHTTP(0x9019c0, 0xc420112800, 0xc420112880, 0x7f5771b811e8, 0xc4200980e8, 0xc420138800)
/home/linux/go/src/github.com/urfave/negroni/negroni.go:38 +0xa5
github.com/urfave/negroni.(middleware).ServeHTTP-fm(0x7f5771b811e8, 0xc4200980e8, 0xc420138800)
/home/linux/go/src/github.com/urfave/negroni/negroni.go:38 +0x60
github.com/urfave/negroni.(*Static).ServeHTTP(0xc42008ff20, 0x7f5771b811e8, 0xc4200980e8, 0xc420138800, 0xc420112960)
/home/linux/go/src/github.com/urfave/negroni/static.go:34 +0x8c
github.com/urfave/negroni.middleware.ServeHTTP(0x900e00, 0xc42008ff20, 0xc420112860, 0x7f5771b811e8, 0xc4200980e8, 0xc420138800)
/home/linux/go/src/github.com/urfave/negroni/negroni.go:38 +0xa5
github.com/urfave/negroni.(middleware).ServeHTTP-fm(0x7f5771b811e8, 0xc4200980e8, 0xc420138800)
/home/linux/go/src/github.com/urfave/negroni/negroni.go:38 +0x60
github.com/urfave/negroni.(*Logger).ServeHTTP(0xc42008f9b0, 0x7f5771b811e8, 0xc4200980e8, 0xc420138800, 0xc420112940)
/home/linux/go/src/github.com/urfave/negroni/logger.go:62 +0x8e
github.com/urfave/negroni.middleware.ServeHTTP(0x900da0, 0xc42008f9b0, 0xc420112840, 0x7f5771b811e8, 0xc4200980e8, 0xc420138800)
/home/linux/go/src/github.com/urfave/negroni/negroni.go:38 +0xa5
github.com/urfave/negroni.(middleware).ServeHTTP-fm(0x7f5771b811e8, 0xc4200980e8, 0xc420138800)
/home/linux/go/src/github.com/urfave/negroni/negroni.go:38 +0x60
github.com/urfave/negroni.(*Recovery).ServeHTTP(0xc42009a870, 0x7f5771b811e8, 0xc4200980e8, 0xc420138800, 0xc420112920)
/home/linux/go/src/github.com/urfave/negroni/recovery.go:193 +0x8c
github.com/urfave/negroni.middleware.ServeHTTP(0x900de0, 0xc42009a870, 0xc420112820, 0x7f5771b811e8, 0xc4200980e8, 0xc420138800)
/home/linux/go/src/github.com/urfave/negroni/negroni.go:38 +0xa5
github.com/urfave/negroni.(*Negroni).ServeHTTP(0xc42008ff80, 0x904660, 0xc42012e1c0, 0xc420138800)
/home/linux/go/src/github.com/urfave/negroni/negroni.go:96 +0xf2
net/http.serverHandler.ServeHTTP(0xc420091040, 0x904660, 0xc42012e1c0, 0xc420138800)
/usr/local/go/src/net/http/server.go:2694 +0xbc
net/http.(*conn).serve(0xc4200a74a0, 0x9049a0, 0xc42009c500)
/usr/local/go/src/net/http/server.go:1830 +0x651
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2795 +0x27b
My code:
package models
import (
"database/sql"
_ "github.com/mattn/go-sqlite3"
)
type User struct {
Id int `json:"id"`
User string `json:"user"`
Admin bool `json:"admin"`
}
func PrepareSqlite() (db *sql.DB, err error) {
db, err = sql.Open("sqlite3","./auth.db")
if err != nil {
panic(err)
return
}
defer db.Close()
// My other Laptop fails here
statement, _ := db.Prepare("CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY, username TEXT, passw TEXT, admin INTEGER)")
statement.Exec()
return db, nil
}
func PassQuery(username string) (pass string) {
db, err := PrepareSqlite()
if err != nil {
panic(err)
return
}
// Server fails here
rows, err := db.Query("SELECT passw FROM users WHERE username=\"" + username + "\"")
defer rows.Close()
if err != nil {
panic(err)
return
}
for rows.Next() {
rows.Scan(&pass)
}
return pass
}
Anyone got any idea why this might happen? My machine is running Arch, the server is on Debian. I've already tried using several other gosqlite-drivers. On yet another machine, a similar error occurs, but the stacktrace shows that it occurs in the PrepareSqlite() method when calling db.Prepare(...). By this point I have no idea what this might be.
Edit: I've tried removing defer rows.Close(), which gave me the Error "Database is closed". After that, I've tried opening the database directly in the PassQuery() method, which gave me the first error once again. I suspect it might be something with opening the database, but I'm not sure what.
In your PrepareSqlite method, you are deferring a call to db.Close() which means the database connection is closed once the method returns.
You are then trying to use that closed db connection in the PassQuery, which is causing the panic (you shouldn't try to use a closed db connection).
You should remove the defer db.Close() from the PrepareSqlite method and only call db.Close() once you have finished using the connection.
Here is an extract of my code:
func sendTo(url string, someDataPoints [] DataPoint) {
ro := &grequests.RequestOptions{JSON: someDataPoints, InsecureSkipVerify: false}
grequests.Post(url, ro)
return
}
func forward(someDataPoints []DataPoint) int {
endpoint := "https://example.org"
go sendTo(endpoint, someDataPoints)
}
The function forward is called something like 100 times per second. The program works for thirty hour or so and ends up failing with the following error:
goroutine 265125012 [semacquire, 41 minutes]:
sync.runtime_notifyListWait(0xc4204be2d0, 0x1bb56b6)
/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4204be2c0)
/usr/local/go/src/sync/cond.go:56 +0x80
net/http.(*http2ClientConn).awaitOpenSlotForRequest(0xc420d5e000, 0xc42c167000, 0x0, 0x0)
/usr/local/go/src/net/http/h2_bundle.go:7528 +0x11d
net/http.(*http2ClientConn).roundTrip(0xc420d5e000, 0xc42c167000, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/h2_bundle.go:7336 +0x110
net/http.(*http2Transport).RoundTripOpt(0xc4200100f0, 0xc42c167000, 0xffffffffffffff00, 0x0, 0x40bac0, 0x705e50)
/usr/local/go/src/net/http/h2_bundle.go:6929 +0x150
net/http.(*http2Transport).RoundTrip(0xc4200100f0, 0xc42c167000, 0xc42c1a04c0, 0x0, 0xc42c1a0438)
/usr/local/go/src/net/http/h2_bundle.go:6891 +0x3a
net/http.http2noDialH2RoundTripper.RoundTrip(0xc4200100f0, 0xc42c167000, 0x7fff668abf02, 0x5, 0xc42016a088)
/usr/local/go/src/net/http/h2_bundle.go:991 +0x39
net/http.(*Transport).RoundTrip(0x8aa800, 0xc42c167000, 0x8aa800, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:380 +0xc36
net/http.send(0xc42c167000, 0x745660, 0x8aa800, 0x0, 0x0, 0x0, 0xc44002b628, 0xb000000000411729, 0xc420d52b50, 0x1)
/usr/local/go/src/net/http/client.go:252 +0x185
net/http.(*Client).send(0x8afa20, 0xc42c167000, 0x0, 0x0, 0x0, 0xc44002b628, 0x0, 0x1, 0x698620)
/usr/local/go/src/net/http/client.go:176 +0xfa
net/http.(*Client).Do(0x8afa20, 0xc42c167000, 0x7fff668abf02, 0x3a, 0xc42bcc5b80)
/usr/local/go/src/net/http/client.go:615 +0x28d
github.com/levigross/grequests.buildRequest(0x705a0f, 0x4, 0x7fff668abf02, 0x3a, 0xc42bcc5b80, 0x8afa20, 0x8, 0x18, 0xc431e455a0)
/go/src/github.com/levigross/grequests/request.go:194 +0x189
github.com/levigross/grequests.doRegularRequest(0x705a0f, 0x4, 0x7fff668abf02, 0x3a, 0xc42bcc5b80, 0x690380, 0x1, 0xc431e455a0)
/go/src/github.com/levigross/grequests/request.go:136 +0x6f
github.com/levigross/grequests.Post(0x7fff668abf02, 0x3a, 0xc42bcc5b80, 0xc431e455a0, 0x4, 0xc42c1703c0)
/go/src/github.com/levigross/grequests/base.go:41 +0x54
main.sendTo(0x7fff668abf02, 0x3a, 0xc42bcd33b0, 0x1, 0x1, 0x0, 0x0)
/go/src/github.com/SomeCompany/SomeProduct/cmd/SomeCli/main.go:66 +0xdd
created by main.forward
/go/src/github.com/SomeCompany/SomeProduct/cmd/SomeCli/main.go:80 +0xc8
What could explain this issue?
It does not seem to have an actual error displayed in there. Just a stack trace. I'm not sure why.
What I do see is that you appear to just launch goroutines without any limit. What does your program do if it cannot forward and complete sendTo requests faster than they arrive?
I think that at some point Go will run out of memory for goroutine stacks, or something else will overflow. I'd expect some sort of "out of memory" error, but I don't see one there.
Nevertheless, that is my best guess: some sort of resource exhaustion caused by creating more goroutines than are finishing.