Why does WTForms have unbound fields the first time the page is loaded? - flask-wtforms

I have the following form classes:
class FieldsRequiredForm(FlaskForm):
class Meta:
def render_field(self, field, render_kw):
render_kw.setdefault('required', True)
return super().render_field(field, render_kw)
class SingleStringFieldForm(FieldsRequiredForm):
def __init__(self, field_label=None, question_id=None,
submit_label='Submit'):
super().__init__()
SingleStringFieldForm.answer = StringField(field_label)
SingleStringFieldForm.question_id = HiddenField(default=question_id)
SingleStringFieldForm.submit = SubmitField(submit_label)
class SingleRadioFieldForm(FieldsRequiredForm):
def __init__(self, field_label=None, question_id=None,
submit_label='Submit', choices=None):
super().__init__()
SingleRadioFieldForm.answer = RadioField(field_label, choices=choices)
SingleRadioFieldForm.question_id = HiddenField(default=question_id)
SingleRadioFieldForm.submit = SubmitField(submit_label)
The function that's using these forms looks like this:
#bp.route('/survey/<string:slug>', methods=['GET', 'POST'])
def question(slug):
survey = Survey.query.filter_by(slug=slug).first_or_404()
questions = survey.questions
question_ids = [question.id for question in questions]
if 'answers' not in session:
session['answers'] = json.dumps({id: None for id in question_ids})
answers = json.loads(session['answers'])
if request.method == 'POST':
record_submitted_answer()
answers = json.loads(session['answers'])
if None in answers.values():
question = get_next_question()
if question.category == 'word':
form = SingleStringFieldForm(field_label=question.question,
question_id=question.id)
elif question.category == 'likert':
form = SingleRadioFieldForm(field_label=question.question,
question_id=question.id,
choices=tuple(likert().items()))
else:
form = SingleStringFieldForm()
if form.validate_on_submit():
if None not in answers.values():
write_answers_to_database(survey=survey)
with open('app/static/ty.txt', 'r') as f:
ty = [x.strip() for x in f.readlines()]
return render_template('ty.html', ty=ty)
return redirect(url_for('survey.question', slug=slug))
return render_template('survey.html', form=form, answers=answers)
The first time I load the page after clearing the session, the form doesn't show up, and when I'm stepping through with a debugger when that happens, the debugger reports that form.answer has a value of:
<UnboundField(RadioField, ('Question 1',), {'choices': (('1', 'Strongly Agree'),
('2', 'Agree'), ('3', 'Neutral'), ('4', 'Disagree'), ('5', 'Strongly Disagree'))})>
If I reload the page, it has this value:
<app.survey.forms.SingleRadioFieldForm object at 0x110788d30>
I don't notice anything else different in the state of the program between the two page loads.
What is causing this the first time and how can I fix it?

While I still don't know why the original function wasn't working on first page load, but did on a reload, I went about this in another way and solved the problem. Instead of setting the fields within __init__, I subclassed within the function and edited the class directly.
#bp.route('/survey/<string:slug>', methods=['GET', 'POST'])
def survey(slug):
class F(FieldsRequiredForm):
pass
...
if None in answers.values():
question = get_next_question()
F.question_id = HiddenField(default=question.id)
if question.category == 'word':
F.answer = StringField(question.question)
elif question.category == 'likert':
F.answer = RadioField(question.question, choices=tuple(likert().items()))
F.submit = SubmitField('Submit')
else:
F.answer = StringField()
form = F()
...

Related

How can I use Google Cloud Functions to run a web scraper?

Thanks in advance for your help.
I'm currently running a webscraper - this is the first time I've ever done something like this - It pulls addresses from the URL and then will match the address to the users input. This will be going into a chat bot, I wondering how I can make this run on Google Functions. Whats the process to do this, is there a tutorial anywhere?
This is my code so far. There is a small items file too
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from ..items import DataItem
from fuzzywuzzy import fuzz
from urllib.parse import urljoin
import scrapy
class AddressesSpider(scrapy.Spider):
name = 'Addresses'
allowed_domains = ['find-energy-certificate.service.gov.uk']
postcode = "bh10+4ah"
start_urls = ['https://find-energy-certificate.service.gov.uk/find-a-certificate/search-by-postcode?postcode=' + postcode]
## def start_requests(self):
## self.first = input("Please enter the address you would like to match: ")
## yield scrapy.Request(url=self.start_urls[0], callback=self.parse)
def parse(self, response):
first = input("Please enter the address you would like to match: ")
highest_ratios = []
highest_item = None
for row in response.xpath('//table[#class="govuk-table"]//tr'):
address = row.xpath("normalize-space(.//a[#class='govuk-link']/text())").extract()[0].lower()
address = address.rsplit(',', 2)[0]
link = row.xpath('.//a[#class="govuk-link"]/#href').extract()
details = row.xpath("normalize-space(.//td/following-sibling::td)").extract()
ratio = fuzz.token_set_ratio(address, first)
item = DataItem()
item['link'] = link
item['details'] = details
item['address'] = address
item['ratioresult'] = ratio
if len(highest_ratios) < 3:
highest_ratios.append(item)
elif ratio > min(highest_ratios, key=lambda x: x['ratioresult'])['ratioresult']:
highest_ratios.remove(min(highest_ratios, key=lambda x: x['ratioresult']))
highest_ratios.append(item)
highest_ratios_100 = [item for item in highest_ratios if item['ratioresult'] == 100]
if highest_ratios_100:
for item in highest_ratios_100:
yield item
else:
yield max(highest_ratios, key=lambda x: x['ratioresult'])
if len(highest_ratios_100) > 1:
for i, item in enumerate(highest_ratios_100):
print(f"{i+1}: {item['address']}")
selected = int(input("Please select the correct address by entering the number corresponding to the address: ")) - 1
selected_item = highest_ratios_100[selected]
else:
selected_item = highest_ratios_100[0] if highest_ratios_100 else max(highest_ratios, key=lambda x: x['ratioresult'])
new_url = selected_item['link'][0]
new_url = str(new_url)
if new_url:
base_url = 'https://find-energy-certificate.service.gov.uk'
print(f'Base URL: {base_url}')
print(f'New URL: {new_url}')
new_url = urljoin(base_url, new_url)
print(f'Combined URL: {new_url}')
yield scrapy.Request(new_url, callback=self.parse_new_page)
def parse_new_page(self, response):
Postcode = response.xpath('normalize-space((//p[#class="epc-address govuk-body"]/text())[last()])').extract()
Town = response.xpath('normalize-space((//p[#class="epc-address govuk-body"]/text())[last()-1])').extract()
First = response.xpath(".//p[#class='epc-address govuk-body']").extract()
Type = response.xpath('normalize-space(//dd[1]/text())').extract_first()
Walls = response.xpath("//th[contains(text(), 'Wall')]/following-sibling::td[1]/text()").extract()
Roof = response.xpath("//th[contains(text(), 'Roof')]/following-sibling::td[1]/text()").extract()
Heating = response.xpath("//th[text()='Main heating']/following-sibling::td[1]/text()").extract_first()
CurrentScore = response.xpath('//body[1]/div[2]/main[1]/div[1]/div[3]/div[3]/svg[1]/svg[1]/text[1]/text()').re_first("[0-9+]{1,2}")
Maxscore = response.xpath('//body[1]/div[2]/main[1]/div[1]/div[3]/div[3]/svg[1]/svg[2]/text[1]/text()').re_first("[0-9+]{2}")
Expiry = response.xpath('normalize-space(//b)').extract_first()
FloorArea = response.xpath('//dt[contains(text(), "floor area")]/following-sibling::dd/text()').re_first("[0-9+]{2,3}")
Steps = response.xpath("//h3[contains(text(),'Step')]/text()").extract()
yield {
'Postcode': Postcode,
'Town': Town,
'First': First,
'Type': Type,
'Walls': Walls,
'Roof': Roof,
'Heating': Heating,
'CurrentScore': CurrentScore,
'Maxscore': Maxscore,
'Expiry': Expiry,
'FloorArea': FloorArea,
'Steps': Steps
}
I've tried googling and having a look around and can't get how to deploy this as a project to run on google functions or can I just copy the code into the console somewhere?
You can try running your spider from a script. However, a better solution would be to wrap scrapy in its own child process.
For example:
from multiprocessing import Process, Queue
from ... import MySpider
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
def my_cloud_function(event, context):
def script(queue):
try:
settings = get_project_settings()
settings.setdict({
'LOG_LEVEL': 'ERROR',
'LOG_ENABLED': True,
})
process = CrawlerProcess(settings)
process.crawl(MySpider)
process.start()
queue.put(None)
except Exception as e:
queue.put(e)
queue = Queue()
# wrap the spider in a child process
main_process = Process(target=script, args=(queue,))
main_process.start() # start the process
main_process.join() # block until the spider finishes
result = queue.get() # check the process did not return an error
if result is not None:
raise result
return 'ok'
You can refer to this tutorial for more info.

Control display of an ipyvuetify page by a dropdown works in notebook not in voila

I have encountered another working in notebook but not in voila issue. I have tried for a couple of hours but feel like I am still missing something and therefore seeking expert opinions here.
I have a function create_pages_and_run() that takes in a dictionary, d, as an input to generate a dashboard (the data type is ipyvuetify.generated.App.App). The dictionary can be retrieved from a json file scenario_dict using a country name as key where I designed a dropdown to collect the country name.
The purpose is to ask user to select a country name and the page will be redrawn/refreshed. I have the following code that works in notebook but not in Voila. (Works means that when a new country name is selected, the dashboard is displayed with the widgets using data from that countries)
scenario_dropdown = widgets.Dropdown(
options=all_scenarios,
value=initial_scenario,
description="Scenario",
layout=widgets.Layout(margin="0 20px 0 0", height="39px", width="15%"),
)
d = scenario_dict[initial_scenario]
app = create_pages_and_run(d)
#the below code works for notebook
def on_change(change):
global d, app
if change["name"] == "value" and (change["new"] != change["old"]):
d = scenario_dict[change["new"]]
app = create_pages_and_run(d)
clear_output()
display(app)
scenario_dropdown.observe(on_change)
My failed code using ipywidgets.Output is as below. (Failed in the sense, after selecting country name in the dropdown no change is observed).
scenario_dropdown = widgets.Dropdown(
options=all_scenarios,
value=initial_scenario,
description="Scenario",
layout=widgets.Layout(margin="0 20px 0 0", height="39px", width="15%"),
)
d = scenario_dict[initial_scenario]
app = create_pages_and_run(d)
out = widgets.Output()
with out:
display(app)
# the code works failed for voila
def on_change(change):
global d, app, out
if change["name"] == "value" and (change["new"] != change["old"]):
d = scenario_dict[change["new"]]
app = create_pages_and_run(d)
out.clear_output()
with out:
display(app)
display(out)
scenario_dropdown.observe(on_change)
I appreciate your help, thanks.
I'm not sure why your code didn't work. Maybe it was the use of globals which can be avoided. Could you provide a working example to test. Here is a working example based on your code that works in Voila.
import ipywidgets as widgets
all_scenarios = ['aa','bb','cc']
initial_scenario = all_scenarios[0]
scenario_dict = {}
scenario_dict['aa'] = 'do_this'
scenario_dict['bb'] = 'do_that'
scenario_dict['cc'] = 'do_what'
def create_pages_and_run(action):
print(action)
scenario_dropdown = widgets.Dropdown(
options=all_scenarios,
value=initial_scenario,
description="Scenario",
layout=widgets.Layout(margin="0 20px 0 0", height="39px", width="15%"),
)
d = scenario_dict[initial_scenario]
out = widgets.Output()
with out:
create_pages_and_run(d)
app = widgets.VBox([scenario_dropdown, out])
def on_change(change):
if change["name"] == "value" and (change["new"] != change["old"]):
d = scenario_dict[change["new"]]
out.clear_output()
with out:
create_pages_and_run(d)
scenario_dropdown.observe(on_change)
app

Tkinter create RadioButtons from dictionary issue

Here is my problem: I've made a tkinter program which calculate the cost of a creation my wife make. She sews accessories. So far my program is working pretty well. I search in a JSON file if this fabric already exist in some kind of catalogue.
When my catalogue have the same fabric name, I want to make a popup window with radiobutton. By checking the correct radiobutton I chose the correct fabric. So I've extracted the dictionary of same fabric names, when I type the fabric name entry and then click on enter, it sends the dictionnary to the popup window which creates radiobutton in a for loop.
The problem comes here, my radiobutton are correctly made but impossible to get there state nor which button is checked...
First, here's the dictionary :
{"Tissu 1": {"Type": "Tissu", "Matière": "coton", "Prix du coupon": "8",
"Laise du coupon (cm)": "150",
"Prix au mètre (€)": 8.0,
"Fournisseur": "lol"},
"Tissu 2": {"Type": "Tissu", "Matière": "molleton", "Prix du coupon": "10",
"Laise du coupon (cm)": "150",
"Prix au mètre (€)": 10.0,
"Fournisseur": "lol"},
"Tissu 3": {"Type": "Tissu", "Matière": "coton", "Prix du coupon": "12",
"Laise du coupon (cm)": "150",
"Prix au mètre (€)": 12.0,
"Fournisseur": "mol"}}
Here's my code:
First there is the main class:
class ProgramCalcul(tk.Tk):
def __init__(self, *args, **kwargs):
self.frames = {}
for F in (StartPage, CalculTissuPage, CalculMercPage):
frame = F(container, self)
self.frames[F] = frame
frame.grid(row = 0, column = 0, sticky = "nsew")
self.show_frameSP(StartPage)
def show_frameSP(self, cont):
frame = self.frames[cont]
frame.tkraise()
Then I have a class for each other calcul page
Here is the fabric calcul page (I've removed some entries and other widgets for legibility and because they work fine):
class CalculTissuPage(tk.Frame):
def __init__(self, parent, controller):
tk.Frame.__init__(self, parent)
tissuType = tk.StringVar()
TissuNameEntry = tk.Entry(self, textvariable = tissuType, width = 50)
TissuNameEntry.grid(row = 2, padx = 20, pady = 5, sticky = "ew")
def keyPress(arg):
checkCatalogueTissu(TissuNameEntry.get())
TissuNameEntry.bind('<Return>', keyPress)
def checkCatalogueTissu(txt):
tmpDict = {}
with open("catalogue tissu.json") as json_file:
data = json.load(json_file)
dictLen = len(data)
for i in range(1, dictLen+1):
if "Tissu {}".format(i) not in data:
pass
else:
if data["Tissu {}".format(i)]["Matière"] == txt:
tmpDict["Tissu {}".format(i)] = data["Tissu {}".format(i)]
popupRadio(tmpDict)
def setText(txtPrix, txtFournisseur, txtLaise, txtLong):
couponPrixEntry.delete(0,tk.END)
couponPrixEntry.insert(0,txtPrix)
fournisseurEntry.delete(0, tk.END)
fournisseurEntry.insert(0, txtFournisseur)
laiseDimEntry.delete(0, tk.END)
laiseDimEntry.insert(0, txtLaise)
longCouponEntry.delete(0,tk.END)
longCouponEntry.insert(0,txtLong)
And finally the function which create the popup in which appear the radiobuttons:
def popupRadio(tmpDict):
popup = tk.Toplevel()
popup.wm_title("!")
popupVar = tk.IntVar()
def checkState(popupVar):
print(popupVar.get())
for (key, val) in tmpDict.items():
tk.Radiobutton(popup, text = key, variable = popupVar, value = val, command = checkState(popupVar)).grid()
print(key, popupVar.get())
chk = ttk.Button(popup, text = "Select this", command = checkState(popupVar))
chk.grid()
popup.mainloop()
When I run it, the checkState() method seems to run automatically and then the chk button doesn't do anything... I can check the radiobuttons but nothing happens (even if I pass the command=chechState in the radiobutton settings. I want to get which button is checked to use this value in my setText() method further.
for the moment, here is the output the print() send in my console :
0
Tissu 1 0
0
Tissu 3 0
0
So here I am, it's been 2 days since I've tried evrything I found on the web but nothing...
A little help would be very appreciated
PS: sorry if my english is not perfect :)

Fetch datastore entity by id inside of a Dataflow transform

I have 2 datastore models:
class KindA(ndb.Model):
field_a1 = ndb.StringProperty()
field_a2 = ndb.StringProperty()
class KindB(ndb.Model):
field_b1 = ndb.StringProperty()
field_b2 = ndb.StringProperty()
key_to_kind_a = ndb.KeyProperty(KindA)
I want to query KindB and output it to a csv file, but if an entity of KindB points to an entity in KindA I want those fields to be present in the csv as well.
If I was able to use ndb inside of a transform I would setup my pipeline like this
def format(element): # element is an `entity_pb2` object of KindB
try:
obj_a_key_id = element.properties.get('key_to_kind_a', None).key_value.path[0]
except:
obj_a_key_id = None
# <<<<<<<<<<<<<<<<<<<<<<<<<<<<<< HOW DO I DO THIS
obj_a = ndb.Key(KindA, obj_a_key_id).get() if obj_a_key_id else None
return ",".join([
element.properties.get('field_b1', None).string_value,
element.properties.get('field_b2', None).string_value,
obj_a.properties.get('field_a1', None).string_value if obj_a else '',
obj_a.properties.get('field_a2', None).string_value if obj_a else '',
]
def build_pipeline(project, start_date, end_date, export_path):
query = query_pb2.Query()
query.kind.add().name = 'KindB'
filter_1 = datastore_helper.set_property_filter(query_pb2.Filter(), 'field_b1', PropertyFilter.GREATER_THAN, start_date)
filter_2 = datastore_helper.set_property_filter(query_pb2.Filter(), 'field_b1', PropertyFilter.LESS_THAN, end_date)
datastore_helper.set_composite_filter(query.filter, CompositeFilter.AND, filter_1, filter_2)
p = beam.Pipeline(options=pipeline_options)
_ = (p
| 'read from datastore' >> ReadFromDatastore(project, query, None)
| 'format' >> beam.Map(format)
| 'write' >> apache_beam.io.WriteToText(
file_path_prefix=export_path,
file_name_suffix='.csv',
header='field_b1,field_b2,field_a1,field_a2',
num_shards=1)
)
return p
I suppose I could use ReadFromDatastore to query all entities of KindA and then use CoGroupByKey to merge them, but KindA has millions of records and that would be very inefficient.
Per the reccommendations in this answer: https://stackoverflow.com/a/49130224/4458510
I created the following utils, which were inspired by the source code of
DatastoreWriteFn in apache_beam.io.gcp.datastore.v1.datastoreio
write_mutations and fetch_entities in apache_beam.io.gcp.datastore.v1.helper
import logging
import time
from socket import error as _socket_error
from apache_beam.metrics import Metrics
from apache_beam.transforms import DoFn, window
from apache_beam.utils import retry
from apache_beam.io.gcp.datastore.v1.adaptive_throttler import AdaptiveThrottler
from apache_beam.io.gcp.datastore.v1.helper import make_partition, retry_on_rpc_error, get_datastore
from apache_beam.io.gcp.datastore.v1.util import MovingSum
from apache_beam.utils.windowed_value import WindowedValue
from google.cloud.proto.datastore.v1 import datastore_pb2, query_pb2
from googledatastore.connection import Datastore, RPCError
_WRITE_BATCH_INITIAL_SIZE = 200
_WRITE_BATCH_MAX_SIZE = 500
_WRITE_BATCH_MIN_SIZE = 10
_WRITE_BATCH_TARGET_LATENCY_MS = 5000
def _fetch_keys(project_id, keys, datastore, throttler, rpc_stats_callback=None, throttle_delay=1):
req = datastore_pb2.LookupRequest()
req.project_id = project_id
for key in keys:
req.keys.add().CopyFrom(key)
#retry.with_exponential_backoff(num_retries=5, retry_filter=retry_on_rpc_error)
def run(request):
# Client-side throttling.
while throttler.throttle_request(time.time() * 1000):
logging.info("Delaying request for %ds due to previous failures", throttle_delay)
time.sleep(throttle_delay)
if rpc_stats_callback:
rpc_stats_callback(throttled_secs=throttle_delay)
try:
start_time = time.time()
response = datastore.lookup(request)
end_time = time.time()
if rpc_stats_callback:
rpc_stats_callback(successes=1)
throttler.successful_request(start_time * 1000)
commit_time_ms = int((end_time - start_time) * 1000)
return response, commit_time_ms
except (RPCError, _socket_error):
if rpc_stats_callback:
rpc_stats_callback(errors=1)
raise
return run(req)
# Copied from _DynamicBatchSizer in apache_beam.io.gcp.datastore.v1.datastoreio
class _DynamicBatchSizer(object):
"""Determines request sizes for future Datastore RPCS."""
def __init__(self):
self._commit_time_per_entity_ms = MovingSum(window_ms=120000, bucket_ms=10000)
def get_batch_size(self, now):
"""Returns the recommended size for datastore RPCs at this time."""
if not self._commit_time_per_entity_ms.has_data(now):
return _WRITE_BATCH_INITIAL_SIZE
recent_mean_latency_ms = (self._commit_time_per_entity_ms.sum(now) / self._commit_time_per_entity_ms.count(now))
return max(_WRITE_BATCH_MIN_SIZE,
min(_WRITE_BATCH_MAX_SIZE,
_WRITE_BATCH_TARGET_LATENCY_MS / max(recent_mean_latency_ms, 1)))
def report_latency(self, now, latency_ms, num_mutations):
"""Reports the latency of an RPC to Datastore.
Args:
now: double, completion time of the RPC as seconds since the epoch.
latency_ms: double, the observed latency in milliseconds for this RPC.
num_mutations: int, number of mutations contained in the RPC.
"""
self._commit_time_per_entity_ms.add(now, latency_ms / num_mutations)
class LookupKeysFn(DoFn):
"""A `DoFn` that looks up keys in the Datastore."""
def __init__(self, project_id, fixed_batch_size=None):
self._project_id = project_id
self._datastore = None
self._fixed_batch_size = fixed_batch_size
self._rpc_successes = Metrics.counter(self.__class__, "datastoreRpcSuccesses")
self._rpc_errors = Metrics.counter(self.__class__, "datastoreRpcErrors")
self._throttled_secs = Metrics.counter(self.__class__, "cumulativeThrottlingSeconds")
self._throttler = AdaptiveThrottler(window_ms=120000, bucket_ms=1000, overload_ratio=1.25)
self._elements = []
self._batch_sizer = None
self._target_batch_size = None
def _update_rpc_stats(self, successes=0, errors=0, throttled_secs=0):
"""Callback function, called by _fetch_keys()"""
self._rpc_successes.inc(successes)
self._rpc_errors.inc(errors)
self._throttled_secs.inc(throttled_secs)
def start_bundle(self):
"""(re)initialize: connection with datastore, _DynamicBatchSizer obj"""
self._elements = []
self._datastore = get_datastore(self._project_id)
if self._fixed_batch_size:
self._target_batch_size = self._fixed_batch_size
else:
self._batch_sizer = _DynamicBatchSizer()
self._target_batch_size = self._batch_sizer.get_batch_size(time.time()*1000)
def process(self, element):
"""Collect elements and process them as a batch"""
self._elements.append(element)
if len(self._elements) >= self._target_batch_size:
return self._flush_batch()
def finish_bundle(self):
"""Flush any remaining elements"""
if self._elements:
objs = self._flush_batch()
for obj in objs:
yield WindowedValue(obj, window.MAX_TIMESTAMP, [window.GlobalWindow()])
def _flush_batch(self):
"""Fetch all of the collected keys from datastore"""
response, latency_ms = _fetch_keys(
project_id=self._project_id,
keys=self._elements,
datastore=self._datastore,
throttler=self._throttler,
rpc_stats_callback=self._update_rpc_stats)
logging.info("Successfully read %d keys in %dms.", len(self._elements), latency_ms)
if not self._fixed_batch_size:
now = time.time()*1000
self._batch_sizer.report_latency(now, latency_ms, len(self._elements))
self._target_batch_size = self._batch_sizer.get_batch_size(now)
self._elements = []
return [entity_result.entity for entity_result in response.found]
class LookupEntityFieldFn(LookupKeysFn):
"""
Looks-up a field on an EntityPb2 object
Expects a EntityPb2 object as input
Outputs a tuple, where the first element is the input object and the second element is the object found during the
lookup
"""
def __init__(self, project_id, field_name, fixed_batch_size=None):
super(LookupEntityFieldFn, self).__init__(project_id=project_id, fixed_batch_size=fixed_batch_size)
self._field_name = field_name
#staticmethod
def _pb2_key_value_to_tuple(kv):
"""Converts a key_value object into a tuple, so that it can be a dictionary key"""
path = []
for p in kv.path:
path.append(p.name)
path.append(p.id)
return tuple(path)
def _flush_batch(self):
_elements = self._elements
keys_to_fetch = []
for element in self._elements:
kv = element.properties.get(self._field_name, None)
if kv and kv.key_value and kv.key_value.path:
keys_to_fetch.append(kv.key_value)
self._elements = keys_to_fetch
read_keys = super(LookupEntityFieldFn, self)._flush_batch()
_by_key = {self._pb2_key_value_to_tuple(entity.key): entity for entity in read_keys}
output_pairs = []
for input_obj in _elements:
kv = input_obj.properties.get(self._field_name, None)
output_obj = None
if kv and kv.key_value and kv.key_value.path:
output_obj = _by_key.get(self._pb2_key_value_to_tuple(kv.key_value), None)
output_pairs.append((input_obj, output_obj))
return output_pairs
The Key to this is the line response = datastore.lookup(request), where:
datastore = get_datastore(project_id) (from apache_beam.io.gcp.datastore.v1.helper.get_datastore)
request is a LookupRequest from google.cloud.proto.datastore.v1.datastore_pb2
response is LookupResponse from google.cloud.proto.datastore.v1.datastore_pb2
The rest of the above code does things like:
using a single connection to the datastore for a dofn bundle
batches keys together before performing a lookup request
throttles interactions with the datastore if requests start to fail
(honestly I don't know how critical these bits are, I just came across them when browsing the apache_beam source code)
The resulting util function LookupEntityFieldFn(project_id, field_name) is a DoFn that takes in an entity_pb2 object as input, extracts and fetches/gets the key_property that resides on the field field_name, and outputs the result as a tuple (the fetch-result is paired with the input object)
My Pipeline code then became
def format(element): # element is a tuple `entity_pb2` objects
kind_b_element, kind_a_element = element
return ",".join([
kind_b_element.properties.get('field_b1', None).string_value,
kind_b_element.properties.get('field_b2', None).string_value,
kind_a_element.properties.get('field_a1', None).string_value if kind_a_element else '',
kind_a_element.properties.get('field_a2', None).string_value if kind_a_element else '',
]
def build_pipeline(project, start_date, end_date, export_path):
query = query_pb2.Query()
query.kind.add().name = 'KindB'
filter_1 = datastore_helper.set_property_filter(query_pb2.Filter(), 'field_b1', PropertyFilter.GREATER_THAN, start_date)
filter_2 = datastore_helper.set_property_filter(query_pb2.Filter(), 'field_b1', PropertyFilter.LESS_THAN, end_date)
datastore_helper.set_composite_filter(query.filter, CompositeFilter.AND, filter_1, filter_2)
p = beam.Pipeline(options=pipeline_options)
_ = (p
| 'read from datastore' >> ReadFromDatastore(project, query, None)
| 'extract field' >> apache_beam.ParDo(LookupEntityFieldFn(project_id=project, field_name='key_to_kind_a'))
| 'format' >> beam.Map(format)
| 'write' >> apache_beam.io.WriteToText(
file_path_prefix=export_path,
file_name_suffix='.csv',
header='field_b1,field_b2,field_a1,field_a2',
num_shards=1)
)
return p

wxpython wx.combobox save wx.StaticText view help me

import wx
import sqlite3
class Frame(wx.Frame):
def __init__(self):
wx.Frame.__init__(self, None)
self.panel = wx.Panel(self)
self.text = wx.StaticText(self.panel)
self.conn = sqlite3.connect("test.db")
self.cursor = self.conn.cursor()
self.autoRefersh()
def autoRefersh(self):
self.LoadList()
wx.CallLater(1000, self.autoRefersh)
def LoadList(self):
self.cursor.execute("SELECT *FROM CLINIC1")
for date1 in self.cursor: pass
self.staticText2_1 = wx.StaticText(self.panel, label=date1[1], style=wx.ALIGN_CENTER, pos=(100,100))
if __name__ == '__main__':
app = wx.App()
frame = Frame()
frame.Show()
app.MainLoop()
combobox data sqlite3 save in why panel show Why it looks different bug??
I do not know why this is happening.
You missed one crucial step, getting the data itself.
You are using the cursor object not the data returned by the cursor.
def LoadList(self):
self.cursor.execute("SELECT *FROM CLINIC1")
data = self.cursor.fetchall()
for date1 in data: pass
self.staticText2_1 = wx.StaticText(self.panel, label=date1[1], style=wx.ALIGN_CENTER, pos=(100,100))
AS you are "passing" in your for loop perhaps what you actually want is only a single record, in which case
data = self.cursor.fetchone()
and drop the for loop
Even better, read the tutorial
https://www.blog.pythonlibrary.org/2012/07/18/python-a-simple-step-by-step-sqlite-tutorial/
In the heading of your question you mention combobox, so I assume that you want to replace the statictext with a combobox. The following should get you started, I'll leave the wx.EVT_COMBOBOX event binding for you to add, as you will need it to do something when you select an item.
import wx
import sqlite3
class Frame(wx.Frame):
def __init__(self):
wx.Frame.__init__(self, None)
self.selected_data=[]
self.panel = wx.Panel(self)
self.combo = wx.ComboBox(self.panel,-1,choices=self.selected_data, size=(130,30))
self.conn = sqlite3.connect("test.db")
self.cursor = self.conn.cursor()
self.combo.SetValue("Choose an Item")
self.autoRefresh()
def autoRefresh(self):
self.LoadList()
def LoadList(self):
self.combo.Clear()
self.cursor.execute("SELECT * FROM CLINIC1")
data = self.cursor.fetchall()
for date1 in data:
self.selected_data.append(date1[1])
for i in self.selected_data:
self.combo.Append(i)
if __name__ == '__main__':
app = wx.App()
frame = Frame()
frame.Show()
app.MainLoop()
Edit:
It should look like this.

Resources