So I am making a Class Which can dynamically return a property depending on whether or not a property was accessed on it. It also detects the name of the property when accessed by a class. my class code is as follows
class ConfigItem(object):
value: object
key:str
default: object
__name__:str = None
def __init__(self, default=None):
self.default: type(default) = default
self.value = default
self.key = "default_value"
def __get__(self, instance, owner):
if self.key:
self.value = os.environ.get(self.key,self.default)
else:
self.value = self.default
def __set_name__(self, owner, name):
self.__name__ = name
self.key = name
I want the code to have the following behavior:
when created like this:
a_test_key = ConfigItem('default_value')
a_test_key.key == 'a_test_key' #True
without having to pass the key into the constructor
and when accessed as so:
key_value = a_test_key
returns a_test_key.value
but when accessed any other way such as:
a_test_key.default
a_test_key.key
returns the respected values.
I think the solution has to do with the get(self, instance, owner) method, but I am unsure how to detect if a property has been accessed from ConfigItem.
Any takers on how to solve this problem?
Related
I have posted a discussion question about this here as well https://github.com/apache/airflow/discussions/19868
Is it possible to specify arguments to a custom xcom backend? If I could force a task to return data (pyarrow table/dataset, pandas dataframe) which would save a file in the correct container with a "predictable file location" path, then that would be amazing. A lot of my custom operator code deals with creating the blob_path, saving the blob, and pushing a list of the blob_paths to xcom.
Since I work with many clients, I would prefer to have the data for Client A inside of the client-a container which uses a different SAS
When I save a file I consider that a "stage" of the data so I would prefer to keep it, so ideally I could provide a blob_path which matches the folder structure I generally use
class WasbXComBackend(BaseXCom):
def __init__(
self,
container: str = "airflow-xcom-backend",
path: str = guid(),
partition_columns: Optional[list[str]] = None,
existing_data_behavior: Optional[str] = None,
) -> None:
super().__init__()
self.container = container
self.path = path
self.partition_columns = partition_columns
self.existing_data_behavior = existing_data_behavior
#staticmethod
def serialize_value(self, value: Any):
if isinstance(value, pd.DataFrame):
hook = AzureBlobHook(wasb_conn_id="azure_blob")
with io.StringIO() as buf:
value.to_csv(path_or_buf=buf, index=False)
hook.load_string(
container_name=self.container,
blob_name=f"{self.path}.csv",
string_data=buf.getvalue(),
)
value = f"{self.container}/{self.path}.csv"
elif isinstance(value, pa.Table):
hook = AzureBlobHook(wasb_conn_id="azure_blob")
write_options = ds.ParquetFileFormat().make_write_options(
version="2.6", use_dictionary=True, compression="snappy"
)
written_files = []
ds.write_dataset(
data=value,
schema=value.schema,
base_dir=f"{self.container}/{self.path}",
format="parquet",
partitioning=self.partition_columns,
partitioning_flavor="hive",
existing_data_behavior=self.existing_data_behavior,
basename_template=f"{self.task_id}-{self.ts_nodash}-{{i}}.parquet",
filesystem=hook.create_filesystem(),
file_options=write_options,
file_visitor=lambda x: written_files.append(x.path),
use_threads=True,
max_partitions=2_000,
)
value = written_files
return BaseXCom.serialize_value(value)
#staticmethod
def deserialize_value(self, result) -> Any:
result = BaseXCom.deserialize_value(result)
if isinstance(result, str) and result.endswith(".csv"):
hook = AzureBlobHook(wasb_conn_id="azure_blob")
with io.BytesIO() as input_io:
hook.get_stream(
container_name=self.container,
blob_name=str(self.path),
input_stream=input_io,
)
input_io.seek(0)
return pd.read_csv(input_io)
elif isinstance(result, list) and ".parquet" in result:
hook = AzureBlobHook(wasb_conn_id="azure_blob")
return ds.dataset(
source=result, partitioning="hive", filesystem=hook.create_filesystem()
)
return result
It's not clear exactly what information you want to be able to retrieve to use as part of your "predictable file location". But there is a PR to pass basic things like dag_id, task_id etc on to serialize_value so that you can use them when naming your stored objects.
Until that is merged, you'll have to override BaseXCom.set.
You need to override BaseXCom.set
a working ,in production example
class MyXComBackend(BaseXCom):
#classmethod
#provide_session
def set(cls, key, value, execution_date, task_id, dag_id, session=None):
session.expunge_all()
# logic to use this custom_xcom_backend only with the necessary dag and task
if cls.is_task_to_custom_xcom(dag_id, task_id):
value = cls.custom_backend_saving_fn(value, dag_id, execution_date, task_id)
else:
value = BaseXCom.serialize_value(value)
# remove any duplicate XComs
session.query(cls).filter(
cls.key == key, cls.execution_date == execution_date, cls.task_id == task_id, cls.dag_id == dag_id
).delete()
session.commit()
# insert new XCom
from airflow.models.xcom import XCom # noqa
session.add(XCom(key=key, value=value, execution_date=execution_date, task_id=task_id, dag_id=dag_id))
session.commit()
#staticmethod
def is_task_to_custom_xcom(dag_id: str, task_id: str) -> bool:
return True # custom your logic here if necessary
I have checked a few of the posts having the same issue and it tells me how to do it but it is not working.
My code:
class Product:
def __init__(self,price,prod_id,quantity):
self.price = price
self.prod_id = prod_id
self.quantity = quantity
self.shop_inventory = {}
self.amount = {}
class Inventory(Product):
def update_stock(self):
self.shop_inventory[self.prod_id] = self.quantity
self.amount[self.prod_id] = self.price
def return_stock_item(self):
look_up = input("What item would you like to check? ")
item = self.shop_inventory[look_up.lower()]
price = self.amount[look_up.lower()]
return f"{look_up.capitalize()}: There are {item} in stock and cost ${price} each."
def display_stock(self):
return self.amount.items()
To set the values Ive been using
stock = Inventory(2.34,'apple',5)
stock.update_stock()
It's part of a class and everything else in the class is working right. Just this one function is not working. I have tried this method as well as the update() method and neither are adding new key values. It is only over writing the same first spot. I have an item check in the class for all the dictionary items and it always only returns one [key][value] no matter how many times I change the key name.
Thank you for any help.
I have a problem with copy_relations after publishing a page.
I have a plugin with additional model. This model has ForeignKey to my plugin.
class InfContactForm(CMSPlugin):
name = models.CharField(max_length=50)
def copy_relations(self, oldinstance):
self.inf_contact_form.all().delete()
for inf_contact_form in oldinstance.inf_contact_form.all():
inf_contact_form.pk = None
inf_contact_form.plugin = self
inf_contact_form.save()
class InfContactFormAgreement(models.Model):
inf_contact_form = models.ForeignKey(InfContactForm, related_name="inf_contact_form")
agreement = HTMLField(blank=True, null=True)
The "InfContactFormAgreement" model is then used as stacked inline in "InfContactForm" plugin form.
Like it is written in docs, there is also copy_relations() method but when the page is published, nothing happens. When I get back again to edit mode InfContactFormAgreement is doubled every time I do it.
Here is also my cms_plugins.py file:
class PluginInfContactForm(CMSPluginBase):
render_template = '_contact_form.html'
name = name1
model = InfContactForm
require_parent = False
inlines = [AgreementStackedInline, ]
def render(self, context, instance, placeholder):
context['instance'] = instance
context = super(PluginInfContactForm, self).render(context, instance, placeholder)
agreements = instance.inf_contact_form.all()
context.update({
'agreements': agreements,
})
return context
plugin_pool.register_plugin(PluginInfContactForm)
Thank you for any advice.
instead of:
inf_contact_form.plugin = self
you should use:
inf_contact_form.inf_contact_form = self
and the resulting code should look like this:
def copy_relations(self, oldinstance):
self.inf_contact_form.all().delete()
for inf_contact_form in oldinstance.inf_contact_form.all():
inf_contact_form.pk = None
inf_contact_form.inf_contact_form = self
inf_contact_form.save()
regards
I had a similar situation, the only difference being that my many-to-many relationship was not in a plugin, but an extension (PageExtension).
In my case, #Dariusz solution did not work, and I had to update the implicit "inbetween" model that exists between the Extension model and the Associated model.
My solution (the key being the "through"):
class Extension(PageExtension):
tags_regions = models.ManyToManyField(Region, related_name="articles", blank=True)
def copy_relations(self, oldinstance):
for region_tag in oldinstance.tags_regions.through.objects.filter(extension=oldinstance):
region_tag.extension = self
region_tag.save()
I'm trying to generate dynamic forms using Flask-WTF to create a new product based on some templates. A product will have a list of required key-value pairs based on its type, as well as a list of parts required to build it. The current relevant code looks as follows:
forms.py:
class PartSelectionForm(Form):
selected_part = QuerySelectField('Part', get_label='serial', allow_blank=True)
part_type = StringField('Type')
slot = IntegerField('Slot')
required = BooleanField('Required')
def __init__(self, csrf_enabled=False, *args, **kwargs):
super(PartSelectionForm, self).__init__(csrf_enabled=False, *args, **kwargs)
class NewProductForm(Form):
serial = StringField('Serial', default='', validators=[DataRequired()])
notes = TextAreaField('Notes', default='')
parts = FieldList(FormField(PartSelectionForm))
views.py:
#app.route('/products/new/<prodmodel>', methods=['GET', 'POST'])
#login_required
def new_product(prodmodel):
try:
model = db.session.query(ProdModel).filter(ProdModel.id==prodmodel).one()
except NoResultFound, e:
flash('No products of model type -' + prodmodel + '- found.', 'error')
return redirect(url_for('index'))
keys = db.session.query(ProdTypeTemplate.prod_info_key).filter(ProdTypeTemplate.prod_type_id==model.prod_type_id)\
.order_by(ProdTypeTemplate.prod_info_key).all()
parts_needed = db.session.query(ProdModelTemplate).filter(ProdModelTemplate.prod_model_id==prodmodel)\
.order_by(ProdModelTemplate.part_type_id, ProdModelTemplate.slot).all()
class F(forms.NewProductForm):
pass
for key in keys:
if key.prod_info_key in ['shipped_os','factory_os']:
setattr(F, key.prod_info_key, forms.QuerySelectField(key.prod_info_key, get_label='version'))
else:
setattr(F, key.prod_info_key, forms.StringField(key.prod_info_key, validators=[forms.DataRequired()]))
form = F(request.form)
if request.method == 'GET':
for part in parts_needed:
entry = form.parts.append_entry(forms.PartSelectionForm())
entry.part_type.data=part.part_type_id
entry.slot.data=slot=part.slot
entry.required.data=part.required
entry.selected_part.query = db.session.query(Part).join(PartModel).filter(PartModel.part_type_id==part.part_type_id, Part.status=='inventory')
if form.__contains__('shipped_os'):
form.shipped_os.query = db.session.query(OSVersion).order_by(OSVersion.version)
if form.__contains__('factory_os'):
form.factory_os.query = db.session.query(OSVersion).order_by(OSVersion.version)
if form.validate_on_submit():
...
Everything works as expected on a GET request, but on the validate_on_submit I get errors. The error is that all of the queries and query_factories for the selected_part QuerySelectFields in the list of PartSelectionForms is none, causing either direct errors in WTForms validation code or when Jinja2 attempts to re-render the QuerySelectFields. I'm not sure why this happens on the POST when everything appears to be correct for the GET.
I realized that although I set the required queries on a GET I'm not doing it for any PartSelectionForm selected_part entries on the POST. Since I already intended part_type, slot, and required to be hidden form fields, I added the following immediately before the validate_on_submit and everything works correctly:
for entry in form.parts:
entry.selected_part.query = db.session.query(Part).join(PartModel).\
filter(PartModel.part_type_id==entry.part_type.data, Part.status=='inventory')
I am using QSortFilterProxyModels all the time. However, if a QSqlRelation is setup on the source model, along with a QSqlRelationalDelegate on the view, whenever the view is switched to the proxy model, the QSqlRelationalDelegate disappears, leaving the basic QLineEdit or QSpinBox.
How can I make columns in a view work with both a QSortFilterProxyModel and QSqlRelationalDelegate, giving the expected QCombobox drop down?
This is a better methodology: QSqlRelationalTableModel with QSqlRelationalDelegate not working behind QAbstractProxyModel.
One needs to use mapToSource because the views index can be different then the models index.
class Delegate(QtSql.QSqlRelationalDelegate):
"""
Delegate handles custom editing. This allows the user to have good
editing experience.
Because the join table uses a proxy model a subclass QSqlRelationalDelegate
is required. This is to support the foreign key combobox.
"""
def __init__(self, parent = None):
"""
Class constructor.
"""
# Python super lets you avoid referring to the base class explicitly.
super(Delegate, self).__init__(parent)
def createEditor(self, parent, option, index):
"""
This creates the editors in the delegate.
Reimplemented from QAbstractItemDelegate::createEditor().
Returns the widget used to edit the item specified by
index for editing.
The parent widget and style option are used to control how the
editor widget appears.
1. Get the model associated with the view. In this case it is the
QSortFilterProxyModel.
2. Because with a proxy model the views index does not have to be the
same as the models index. If one sorts,
then the index are not the same.
3. mapToSource.
This is why mapToSource is being used.
mapToSouce Reimplement this function to return the
model index in the proxy model that corresponds to the
sourceIndex from the source model.
4. Return the createEditor with the base index being set to the source
model and not the proxy model.
"""
if index.column() == 2:
proxy = index.model()
base_index = proxy.mapToSource(index)
return super(Delegate, self).createEditor(parent, option, base_index)
else:
return super(Delegate, self).createEditor(parent, option, index)
def setEditorData(self, editor, index):
"""
Once the editor has been created and given to the view
the view calls setEditorData().
This gives the delegate the opportunity to populate the editor
with the current data, ready for the user to edit.
Sets the contents of the given editor to the data for the item
at the given index.
Note that the index contains information about the model being used.
The base implementation does nothing.
If you want custom editing you will need to reimplement this function.
1. Get the model which is a QSortFilterProxyModel.
2. Call mapToSource().
Because with a proxy model the views index does not have to be the
same as the models index. If one sorts,
then the index are not the same.
This is why mapToSource is being used. MapToSouce Reimplement this
function to return the model index in the proxy model
that corresponds to the sourceIndex from the source model.
3. Return setEditorData with the editor and the mapToSource index.
4. Else for all other columns return the base method.
"""
if index.column() == 2:
proxy = index.model()
base_index = proxy.mapToSource(index)
return super(JoinDelegate, self).setEditorData(editor, base_index)
else:
return super(Delegate, self).setEditorData(editor, index)
def setModelData(self, editor, model, index):
if index.column() == 2:
base_model = model.sourceModel()
base_index = model.mapToSource(index)
return super(JoinDelegate, self).setModelData(editor, base_model, base_index)
else:
super(Delegate, self).setModelData(editor, model, index)
def sizeHint(self, option, index):
"""
This pure abstract function must be reimplemented if you want to
provide custom rendering. The options are specified by option and
the model item by index.
"""
if index.isValid():
column = index.column()
text = index.model().data(index)
document = QtGui.QTextDocument()
document.setDefaultFont(option.font)
# change cell Width, height (One can add or subtract to change the relative dimension)
return QtCore.QSize(QtSql.QSqlRelationalDelegate.sizeHint(self, option, index).width() - 200,
QtSql.QSqlRelationalDelegate.sizeHint(self, option, index).height() + 40)
else:
return super(Delegate, self).sizeHint(option, index)
By default, QSqlRelationalDelegate can't handle proxy models, so you have to subclass it. The below is probably far from perfect, so comments/tweaks are welcome, but has been working well on views that have a mixture of QSqlRelations/straight data, without glitches.
class ProxyDelegate(QSqlRelationalDelegate):
def __init__(self):
QSqlRelationalDelegate.__init__(self)
def createEditor(self, p, o, i): # parent, option, index
if i.model().sourceModel().relation(i.column()).isValid(): # if the column has a QSqlRelation, then make the expected QComboBox
e = QComboBox(p)
return e
else:
return QStyledItemDelegate(p).createEditor(p, o, i)
def setEditorData(self, e, i):
m = i.model()
sM = m.sourceModel()
relation = sM.relation(i.column())
if relation.isValid():
m = i.model()
sM = m.sourceModel()
relation = sM.relation(i.column())
pModel = QSqlTableModel() # pModel means populate model. Because I've aimed for generic use, it makes a new QSqlTableModel, even if one already exists elsewhere for that SQL table
pModel.setTable(relation.tableName())
pModel.select()
e.setModel(pModel)
pModel.sort(pModel.fieldIndex(relation.displayColumn()), Qt.AscendingOrder) # default sorting. A custom attribute would need adding to each source model class, in order for this line to know the desired sorting order for this QComboBox delegate
e.setModelColumn(pModel.fieldIndex(relation.displayColumn()))
e.setCurrentIndex(e.findText(m.data(i).toString()))
else:
return QStyledItemDelegate().setEditorData(e, i)
def setModelData(self, e, m, i):
m = i.model() # this could probably be written more elegantly so you don't need to create another SqlModel
sM = m.sourceModel()
relation = sM.relation(i.column())
table = relation.tableName()
indexColumn = relation.indexColumn()
indexColumnId = sM.fieldIndex(indexColumn)
displayColumn = relation.displayColumn()
if relation.isValid():
pModel = QSqlTableModel()
pModel.setTable(relation.tableName())
pModel.select()
displayColumnId = pModel.fieldIndex(displayColumn)
chosenRowInPModel = pModel.match(pModel.index(0, displayColumnId), Qt.DisplayRole, e.currentText())[0].row()
chosenIdInPModel = pModel.data(pModel.index(chosenRowInPModel, indexColumnId)).toString()
m.setData(i, chosenIdInPModel)
self.closeEditor.emit(e, QAbstractItemDelegate.NoHint)
else:
QStyledItemDelegate().setModelData(e, m, i)