If I try to yeild url using response.request.url in my parse() Method, It Returns:
http://192.168.99.100:8050/execute
Returning URL in Lua Script works, but I don't know How can I yield it in parse() method.
import scrapy
from scrapy_splash import SplashRequest
class ComputersSpider(scrapy.Spider):
name = 'computers'
allowed_domains = ['http://daraz.pk']
start_urls = ['http://daraz.pk']
script = '''
function main(splash, args)
splash.private_mode_enabled = false
assert(splash:go(args.url))
assert(splash:wait(1))
input = assert(splash:select("#q"))
input:focus()
input:send_text("computers")
button = assert(splash:select(".search-box__button--1oH7"))
button:mouse_click()
assert(splash:wait(6))
splash:set_viewport_full()
return {
html = splash:html(),
link = splash:url(), -- "I WANT TO YIELD THIS THING IN THE PARSE() METHOD"
}
end '''
def start_requests(self):
for url in self.start_urls:
yield SplashRequest(url = url, callback = self.parse, endpoint= 'execute', args = {"wait" : 3, 'lua_source' : self.script})
def parse(self, response):
link = response.request.url
yield {
'URL' : link,
}
Tried using response.url, it returns the start url
Problem Was solved by replacing link with url in return of lua script:
return {
html = splash:html(),
url = splash:url(), -- "I WANT TO YIELD THIS THING IN THE PARSE() METHOD"
}
And then adding this line in parse method:
yield response.url
Related
I have a ZIO that looks like this:
ZIO[transactor.Transactor[Task], Serializable, String]
and I need to assert the String, that is in that ZIO with another plain String.
So my question is:
How can I assert the plain String, with the String in the ZIO, or
can I lift the String into that same ZIO, to assert it with this one:
ZIO[transactor.Transactor[Task], Serializable, String]
I´m working with ZIO-Test as my testing framework.
I got this based on the rock the jvm zio course. Important things are highlighted in the comments
package com.w33b
import zio.test._
import zio._
object ZIOAssert extends ZIOSpecDefault {
class ATask
abstract class Transact[ATask] {
def get(): String
def put(ATask: ATask): ATask
}
// This is the mock service that we will use to create a ZLayer
val mockTransact = ZIO.succeed(new Transact[ATask] {
override def get(): String = "hello"
override def put(aTask: ATask): ATask = new ATask
})
// This is the method we are testing and we need a service (Zlayer) associated with it.
// We then use the service to invoke a method to get our return value
def methodUnderTest: ZIO[Transact[ATask], Serializable, String] = {
for {
trans <- ZIO.service[Transact[ATask]]
tVal = trans.get()
} yield tVal
}
def spec = suite("Let's test that ZIO lift")(
test("lets test that lift") {
assertZIO(methodUnderTest)(Assertion.equalTo("hello"))
}
).provide(ZLayer.fromZIO(mockTransact)) // Important to provide the layer or we can't test
}
with ZIO 2.x
val spec = suite("tetst")(
test("whatever"){
val effect: ZIO[transactor.Transactor[Task], Serializable, String] = ???
val expected: String = ???
for {
value <- effect
} yield assertTrue(value == expected)
}
)
In general, you just create here a ZIO with the assertion in the success channel.
I am working with Firebase so I have a lot of return types of type ApiFuture<A>. I'd like to turn them into a Task[A] to work with ZIO effects.
We can create a method to convert all of them using Typeclasses:
trait EffectUtils[F[_]] {
def toEffect[A](a: F[A]): Task[A]
}
object EffectUtils {
implicit val apiFuture: EffectUtils[ApiFuture] = new EffectUtils[ApiFuture] {
override def toEffect[A](a: ApiFuture[A]): Task[A] = Task.effectAsync[A]( cb =>
ApiFutures.addCallback(a, new ApiFutureCallback[A] {
override def onFailure(t: Throwable): Unit = cb(Task.fail(t))
override def onSuccess(result: A): Unit = cb(Task.succeed(result))
})
)
}
implicit class ApiFutureOps[A](f: ApiFuture[A]) {
def toEffect(implicit instance: EffectUtils[ApiFuture]) = instance.toEffect(f)
}
}
Now, when we make an API request and want to convert the result to a ZIO type, it's easy:
import EffectUtils._
object App {
// calling a Firebase function
val record: Task[UserRecord] = firebase.getInstance().auth().getUserByEmailAsync(email).toEffect
FastAPI 0.68.0
Python 3.8
from fastapi import Depends, FastAPI, Header, HTTPException
async def verify_key(x_key: str = Header(...)):
if x_key != "fake-super-secret-key":
raise HTTPException(status_code=400, detail="X-Key header invalid")
return x_key
app = FastAPI(dependencies=[Depends(verify_key)])
#app.get("/items/")
async def read_items():
return [{"item": "Portal Gun"}, {"item": "Plumbus"}]
This is a example from FastAPI doucuments (Omit part of the code)
Is there any way to get x_key in read_items()
from fastapi import Request, Depends
async def verify_key(request: Request, x_key: str = Header(...)):
if x_key != "fake-super-secret-key":
raise HTTPException(status_code=400, detail="X-Key header invalid")
# save x_key
request.state.x_key = x_key
return x_key
app = FastAPI(dependencies=[Depends(verify_key)])
#app.get("/items/")
async def read_items(request: Request):
# get 'x_key' from request.state
x_key = request.state.x_key
return [{"item": "Portal Gun"}, {"item": "Plumbus"}]
flask/bottle mimic
# current_request.py
import contextvars
from fastapi import Request
_CURRENT_REQUEST = contextvars.ContextVar("_CURRENT_REQUEST", default=None)
class CurrentRequest:
def __getattr__(self, a):
current_request = _CURRENT_REQUEST.get()
if current_request is None:
raise RuntimeError('Out of context')
return getattr(current_request, a)
request: Request = CurrentRequest()
async def set_current_request(request: Request):
_CURRENT_REQUEST.set(request)
# fastapi app/subroute
from fastapi import Request, Depends
from .current_request import request, set_current_request
async def verify_key(x_key: str = Header(...)):
if x_key != "fake-super-secret-key":
raise HTTPException(status_code=400, detail="X-Key header invalid")
# save x_key
request.state.x_key = x_key
return x_key
# ! set_current_request should go first
app = FastAPI(dependencies=[Depends(set_current_request), Depends(verify_key)])
#app.get("/items/")
async def read_items():
# get 'x_key' from request.state
x_key = request.state.x_key
return [{"item": "Portal Gun"}, {"item": "Plumbus"}]
You can inject verify_key function into read_items as dependency to read its value:
from fastapi import Depends
async def verify_key(x_key: str = Header(...)):
...
#app.get("/items/")
async def read_items(key: str = Depends(verify_key)):
// use key
I need to get response body eg. the response html code from url i am using following code.
location /configure/result.php {
log_by_lua_block {
ngx.log(ngx.ERR, "REQUEST capturing started")
json = require "json"
function getval(v, def)
if v == nil then
return def
end
return v
end
local data = {
request={},
response={}
}
local req = data.request
local resp = data.response
req["host"] = ngx.var.host
req["uri"] = ngx.var.uri
req["headers"] = ngx.req.get_headers()
req["time"] = ngx.req.start_time()
req["method"] = ngx.req.get_method()
req["get_args"] = ngx.req.get_uri_args()
req["post_args"] = ngx.req.get_post_args()
req["body"] = ngx.var.request_body
content_type = getval(ngx.var.CONTENT_TYPE, "")
resp["headers"] = ngx.resp.get_headers()
resp["status"] = ngx.status
resp["duration"] = ngx.var.upstream_response_time
resp["time"] = ngx.now()
resp["body"] = ngx.var.response_body -- Problem Here
ngx.log(ngx.CRIT, json.encode(data));
}
}
But it does not log the response data it recieved from that url eg. the processed source code how could i get the response.data?
My idea is to get response data then use regEx to read specifiq value from the source code which will then do x-y
I cannot test this now, but maybe response is unaccessible during the log phase of request processing?
You can try to get response data within the body_filter_by_lua_block using 'ngx.ctx' table:
body_filter_by_lua_block {
ngx.ctx.response_body = ngx.arg[1]
}
and then use it within the log_by_lua_block block:
log_by_lua_block {
...
resp["body"] = ngx.ctx.response_body
ngx.log(ngx.CRIT, json.encode(data))
}
(please note this is just a guess, please let me know if this works)
I'm including some wordpress contents to my grails app with a customTag
everything works fine.Now i want to render some standard-text if the url's status-code is not 200 OK
i have this so far
def wordpressHp = { body ->
// def url = grailsApplication.config.wordpress.server.url+'?include=true'
def url = "http://www.dasdasdasgsdniga.nd"
def content
try{
content = url.toURL().getText(connectTimeout: 10000)
}catch(e){
content="SORRY WORDPRESS IS CURRENTLY NOT AVAILABLE"
}
out << content
}
The correct url is commented out, i now expect the try to fail.
But instead of faling it's rendering some dns-error page of my provider.
So i think i have to look for http-status code, but how do i do that ?
for any hint, thanks in advance
You could try:
def url = "http://www.dasdasdasgsdniga.nd"
def content
try {
content = url.toURL().openConnection().with { conn ->
readTimeout = 10000
if( responseCode != 200 ) {
throw new Exception( 'Not Ok' )
}
conn.content.withReader { r ->
r.text
}
}
}
catch( e ) {
content="SORRY WORDPRESS IS CURRENTLY NOT AVAILABLE"
}