Currently, I'm developing a custom app. So far I got the DocType ready to be filled in manually. We got files (SQLite3) that I'd like to upload, parse, extract the necessary fields of it and fill in the form. Basically like the import data tool. In my case, no bulk operation is needed and if possible do the extraction part server-side.
What I tried so far
I added a Server Action to call a whitelisted method of my app. I can get the current doc with:
#frappe.whitelist()
def upload_data_and_extract(doc: str):
"""
Uploads and processes an existing file and extracts data from it
"""
doc_dict = json.loads(doc)
custom_dt = frappe.get_doc('CustomDT', doc_dict['name'])
# parse data here
custom_dt.custom_field = "new value from parsed data"
custom_dt.save()
return doc # How do I return a JSON back to the website from the updated doc?
With this approach, I only can do the parsing when the document has been saved before. I'd rather update the fields of the form when the attach field gets modified. Thus, I tried the Server Side Script approach:
frappe.ui.form.on('CustomDT', {
original_data: function(frm, cdt, cdn) {
if(original_data) {
frappe.call({
method: "customapp.customapp.doctype.customdt.customdt.parse_file",
args: {
"doc": frm.doc
},
callback: function(r) {
// code snippet
}
});
}
}
});
Here are my questions:
What's the best approach to upload a file that needs to be parsed to fill the form?
How to access the uploaded file (attachment) the easiest way. (Is there something like frappe.get_attachment()?)
How to refresh the form fields in the callback easily?
I appreciate any help on these topics.
Simon
I have developed the same tool but that was for CSV upload. I am going to share that so it will help you to achieve your result.
JS File.
// Copyright (c) 2020, Bhavesh and contributors
// For license information, please see license.txt
frappe.ui.form.on('Car Upload Tool', {
upload: function(frm) {
frm.call({
doc: frm.doc,
method:"upload_data",
freeze:true,
freeze_message:"Data Uploading ...",
callback:function(r){
console.log(r)
}
})
}
});
Python Code
# -*- coding: utf-8 -*-
# Copyright (c) 2020, Bhavesh and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document
from carrental.carrental.doctype.car_upload_tool.csvtojson import csvtojson
import csv
import json
class CarUploadTool(Document):
def upload_data(self):
_file = frappe.get_doc("File", {"file_url": self.attach_file})
filename = _file.get_full_path()
csv_json = csv_to_json(filename)
make_car(csv_json)
def csv_to_json(csvFilePath):
jsonArray = []
#read csv file
with open(csvFilePath, encoding='latin-1') as csvf:
#load csv file data using csv library's dictionary reader
csvReader = csv.DictReader(csvf,delimiter=";")
#convert each csv row into python dict
for row in csvReader:
frappe.errprint(row)
#add this python dict to json array
jsonArray.append(row)
#convert python jsonArray to JSON String and write to file
return jsonArray
def make_car(car_details):
for row in car_details:
create_brand(row.get('Marke'))
create_car_type(row.get('Fahrzeugkategorie'))
if not frappe.db.exists("Car",row.get('Fahrgestellnr.')):
car_doc = frappe.get_doc(dict(
doctype = "Car",
brand = row.get('Marke'),
model_and_description = row.get('Bezeichnung'),
type_of_fuel = row.get('Motorart'),
color = row.get('Farbe'),
transmission = row.get('Getriebeart'),
horsepower = row.get('Leistung (PS)'),
car_type = row.get('Fahrzeugkategorie'),
car_vin_id = row.get('Fahrgestellnr.'),
licence_plate = row.get('Kennzeichen'),
location_code = row.get('Standort')
))
car_doc.model = car_doc.model_and_description.split(' ')[0] or ''
car_doc.insert(ignore_permissions = True)
else:
car_doc = frappe.get_doc("Car",row.get('Fahrgestellnr.'))
car_doc.brand = row.get('Marke')
car_doc.model_and_description = row.get('Bezeichnung')
car_doc.model = car_doc.model_and_description.split(' ')[0] or ''
car_doc.type_of_fuel = row.get('Motorart')
car_doc.color = row.get('Farbe')
car_doc.transmission = row.get('Getriebeart')
car_doc.horsepower = row.get('Leistung (PS)')
car_doc.car_type = row.get('Fahrzeugkategorie')
car_doc.car_vin_id = row.get('Fahrgestellnr.')
car_doc.licence_plate = row.get('Kennzeichen')
car_doc.location_code = row.get('Standort')
car_doc.save(ignore_permissions = True)
frappe.msgprint("Car Uploaded Successfully")
def create_brand(brand):
if not frappe.db.exists("Brand",brand):
frappe.get_doc(dict(
doctype = "Brand",
brand = brand
)).insert(ignore_permissions = True)
def create_car_type(car_type):
if not frappe.db.exists("Vehicle Type",car_type):
frappe.get_doc(dict(
doctype = "Vehicle Type",
vehicle_type = car_type
)).insert(ignore_permissions = True)
So for this upload tool, I created one single doctype with the below field:
Attach File(Field Type = Attach)
Button (Field Type = Button)
Related
I would like to know how I should change my code to prevent it from Injections:
import sqlite3
def search_in_database(column,searched_data):
con = sqlite3.connect('db.sqlite3')
cursor = con.cursor()
cursor.execute(f"""SELECT
id
FROM
My_library
WHERE
{column} LIKE '%{searched_data}%'
;""")
all = [i for i in cursor.fetchall()]
return all
I found code in web which gives an example of how to do it:
from sqlalchemy.sql import text
# Create a connection conn
stmt = text("""SELECT * FROM users
WHERE user = :username AND password = :password""")
conn.execute(stmt, prams={"username": "foo", "password": "bar"})
but In my HTML file I would like to give to user possibility to choose the:
Place where he wants to search in Titles, authors, published_dates,isbn, language...
and when he choose where He what to search then he types the query.
How to do it in this case, avoiding Injections?
My data base:
class My_library(db.Model):
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String(250))
authors = db.Column(db.String(100))
published_date = db.Column(db.Integer)
isbn_or_identifier = db.Column(db.String(15), unique=True)
page_count = db.Column(db.String(10000))
language = db.Column(db.String(3))
image_links = db.Column(db.String(500))
I also added validators:
from flask_wtf import FlaskForm
from wtforms import SubmitField,StringField
from wtforms.validators import ValidationError,DataRequired,Length, URL
from wtforms.fields.html5 import DateField,IntegerField,DateField,IntegerField, URLField
class AddRecValidators(FlaskForm):
title = StringField(label=('Title:'),validators=[DataRequired(), Length(min=1,max=50)])
authors = StringField(label=('Authors:'),validators=[Length(min=1,max=100)])
published_date = IntegerField(label=('Published date:'),validators=[Length(min=1,max=4)])
isbn_or_identifier = IntegerField(label=('ISBN:'),validators=[Length(min=1,max=15)])
page_count = IntegerField(label=('Page count:'),validators=[ Length(min=1,max=10000)])
language = StringField(label=('Language:'),validators=[ Length(min=1,max=3)])
image_links = URLField(label=('Image links:'))
submit = SubmitField(label=('Add to library'))
Thanks for help in advance :D
You can use sqlalchemy to build the query. For example:
q = My_library.__table__.select().where(
My_library.__table__.c.title == "The searched title"
)
but that's not exactly what you wanted. You can also address the columns by their names like this:
q = My_library.__table__.select().where(
My_library.__table__.c["title"] == "The searched title"
)
# or
q = My_library.__table__.select().where(
My_library.__table__.c["title"].like("%The searched title%")
)
Therefore you can do this:
q = My_library.__table__.select().where(
My_library.__table__.c[column].like(f"%{searched_data}%")
)
cursor.execute(q)
In case you only want the ID, you would do this:
q = sqlalchemy.select([My_library.__table__.c.id]).where(
My_library.__table__.c[column].like(f"%{searched_data}%")
)
# you can print(q) to see what it constructed
cursor.execute(q)
That was SQLAlchemy Query Language. You are using ORM. I suggest you read-up something about a session in flask first.
It is still possible to get to the column-name related attribute and I am not sure this is the most efficient way:
q = session.query(My_library.id).filter(
getattr(My_library, column).like(f"%{searched_data}%"),
)
I'm trying to parse a directory with a collection of xml files from RSS feeds.
I have a similar code for another directory working fine, so I can't figure out the problem. I want to return the items so I can write them to a CSV file. The error I'm getting is:
xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 1, column 0
Here is the site I've collected RSS feeds from: https://www.ba.no/service/rss
It worked fine for: https://www.nrk.no/toppsaker.rss and https://www.vg.no/rss/feed/?limit=10&format=rss&categories=&keywords=
Here is the function for this RSS:
import os
import xml.etree.ElementTree as ET
import csv
def baitem():
basepath = "../data_copy/bergens_avisen"
table = []
for fname in os.listdir(basepath):
if fname != "last_feed.xml":
files = ET.parse(os.path.join(basepath, fname))
root = files.getroot()
items = root.find("channel").findall("item")
#print(items)
for item in items:
date = item.find("pubDate").text
title = item.find("title").text
description = item.find("description").text
link = item.find("link").text
table.append((date, title, description, link))
return table
I tested with print(items) and it returns all the objects.
Can it be how the XML files are written?
Asked a friend and said to test with a try except statement. Found a .DS_Store file, which only applies to Mac computers. I'm providing the solution for those who might experience the same problem in the future.
def baitem():
basepath = "../data_copy/bergens_avisen"
table = []
for fname in os.listdir(basepath):
try:
if fname != "last_feed.xml" and fname != ".DS_Store":
files = ET.parse(os.path.join(basepath, fname))
root = files.getroot()
items = root.find("channel").findall("item")
for item in items:
date = item.find("pubDate").text
title = item.find("title").text
description = item.find("description").text
link = item.find("link").text
table.append((date, title, description, link))
except Exception as e:
print(fname, e)
return table
I've created a dictionary from an Uploaded file in Django.
This dictionary has a nested list of dictionaries:
file = {"name": "filename", "sections": [{"section_name": "string", "lines": [{line_number: 0, "line"; "data"}]}], "etc": "etc"}
The model represents the dictionaries depth too.
class Line(EmbeddedDocument):
line_number = IntField()
line = StringField()
definition = ReferenceField(Definition)
class Section(EmbeddedDocument):
section_name = StringField()
lines = EmbeddedDocumentListField(Line))
class File(Document):
name = StringField()
sections = EmbeddedDocumentListField(Section))
created_on = DateTimeField()
created_by = StringField()
modified_on = DateTimeField()
modified_by = StringField()
In the POST I have the following to chop the file up into the above Dict (the file is a simple text file):
file= {}
with open(os.path.join(path, filename + ".txt"), 'r') as temp_file:
filelines = temp_file.readlines()
sections = []
section = {}
lines = []
for i, l in enumerate(filelines):
if i == 0:
section["section_name"] = "Top"
elif '*' in l:
if l.index('*') == 0 and '*' not in lines[len(lines) - 2"line"]:
section["lines"] = lines
lines = []
sections.append(section)
section = dict()
section["section_name"] = filelines[i + 1][1:-2]
line = {"line_number": i + 1, "line": l}
lines.append(line)
section['lines'] = lines
sections.append(section)
file["name"] = filename
file["sections"] = sections
I will tidy this up eventually.
Once the dict has been made how do I serialise it using the serializer?
Is it possible to insert this into a serializer?
If not how can I get it all into the database with validation?
I've tried json.dumps() and JsonRequst() then putting them in data= for the serializer but get Unable to get repr for <class '....'>
I'm pretty new to Django and MongoDB so if you need more info I can provide :)
Thanks!
Update
Change the model's List Fields to EmbeddedDocumentListField as suggest in the answer.
Answered
Thanks to Boris' suggestion below it pointed me to an error I wasn't getting initially. I had a typo and passing the dict directly into FileSerializer(data=file) works like a charm! :)
James!
The easiest way to validate that your incoming JSONs adhere to the Mongoengine Documents schema that you've specified is to use DRF-Mongoengine's DocumentSerializer.
Basically, what you need to do is create a serializer
serializers.py
import rest_framework_mongoengine
class FileSerializer(rest_framework_mongoengine.DocumentSerializer):
class Meta:
fields = '__all__'
model = File
Then you need a view or viewset that makes use of this Serializer to respond to GET/POST/PUT/DELETE requests.
views.py
from rest_framework_mongoengine import viewsets
class FileViewSet(viewsets.ModelViewSet):
lookup_field = 'id'
serializer_class = FileSerializer
def get_queryset(self):
return File.objects.all()
and register this viewset with a router
urls.py
from rest_framework import routers
# this is DRF router for REST API viewsets
router = routers.DefaultRouter()
# register REST API endpoints with DRF router
router.register(r'file', FileViewSet, r"file")
I'd also recommend using EmbeddedDocumentListField instead of ListField(EmbeddedDocumentField(Section)) - it has additional methods.
I am making a very simple application with 2 webpages at the moment under URLs: localhost:8080/restaurants/ and localhost:8080/restaurants/new.
I have a sqlite database which i manipulate with SQLAlchemy in my python code.
On my first page localhost:8080/restaurants/, this just contains the lists of restaurants available in my database.
My second page localhost:8080/restaurants/new, is where i have a form in order to a new restaurant such that it displays on localhost:8080/restaurants.
However Whenever i enter a new restaurant name on form at localhost:8080/restaurants/new, it fails to redirect me back to localhost:8080/restaurants/ in order to show me the new restaurant, instead it just remains on the same url link localhost:8080/restaurants/new with the message "No data received" .
Below is my code:
import cgi
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
#import libraries and modules
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from database_setup import Base, Restaurant, MenuItem
#create and connect to database
engine = create_engine('sqlite:///restaurantmenu.db')
Base.metadata.bind=engine
DBSession = sessionmaker(bind=engine)
session = DBSession()
class webServerHandler(BaseHTTPRequestHandler):
""" class defined in the main method"""
def do_GET(self):
try:
#look for url then ends with '/hello'
if self.path.endswith("/restaurants"):
self.send_response(200)
#indicate reply in form of html to the client
self.send_header('Content-type', 'text/html')
#indicates end of https headers in the response
self.end_headers()
#obtain all restaurant names from databse
restaurants = session.query(Restaurant).all()
output = ""
output += "<html><body><a href='/restaurants/new'>Add A New Restaurant</a>"
output += "</br></br>"
for restaurant in restaurants:
output += restaurant.name
output += """<div>
<a href='#'>Edit</a>
<a href='#'>Delete</a>
</div>"""
output += "</br></br>"
output += "</body></html>"
self.wfile.write(output)
print output
return
if self.path.endswith("/restaurants/new"):
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
output = ""
output += "<html><body>"
output += "<h1>Add New Restaurant</h1>"
output += "<form method='POST' enctype='multipart/form-data action='/restaurants/new'>"
output += "<input name='newRestaurant' type='text' placeholder='New Restaurant Name'>"
output += "<input name='Create' type='submit' label='Create'>"
output += "</form></body></html>"
self.wfile.write(output)
return
except IOError:
self.send_error(404, "File %s not found" % self.path)
def do_POST(self):
try:
if self.path.endswith("/restaurants/new"):
ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
#check of content-type is form
if ctype == 'mulitpart/form-data':
#collect all fields from form, fields is a dictionary
fields = cgi.parse_multipart(self.rfile, pdict)
#extract the name of the restaurant from the form
messagecontent = fields.get('newRestaurant')
#create the new object
newRestaurantName = Restaurant(name = messagecontent[0])
session.add(newRestaurantName)
session.commit()
self.send_response(301)
self.send_header('Content-type', 'text/html')
self.send_header('Location','/restaurants')
self.end_headers()
except:
pass
def main():
"""An instance of HTTPServer is created in the main method
HTTPServer is built off of a TCP server indicating the
transmission protocol
"""
try:
port = 8080
#server address is tuple & contains host and port number
#host is an empty string in this case
server = HTTPServer(('', port), webServerHandler)
print "Web server running on port %s" % port
#keep server continually listening until interrupt occurs
server.serve_forever()
except KeyboardInterrupt:
print "^C entered, stopping web server...."
#shut down server
server.socket.close()
#run main method
if __name__ == '__main__':
main()
for reference here is my database_setup file where i create the database:
import sys
#importing classes from sqlalchemy module
from sqlalchemy import Column, ForeignKey, Integer, String
#delcaritive_base , used in the configuration
# and class code, used when writing mapper
from sqlalchemy.ext.declarative import declarative_base
#relationship in order to create foreign key relationship
#used when writing the mapper
from sqlalchemy.orm import relationship
#create_engine to used in the configuration code at the
#end of the file
from sqlalchemy import create_engine
#this object will help set up when writing the class code
Base = declarative_base()
class Restaurant(Base):
"""
class Restaurant corresponds to restaurant table
in the database to be created.
table representation for restaurant which
is in the database
"""
__tablename__ = 'restaurant'
#column definitions for the restaurant table
id = Column(Integer, primary_key=True)
name = Column(String(250), nullable=False)
class MenuItem(Base):
"""
class MenuItem corresponds to restaurant table
table representation for menu_item which
is in the database
"""
__tablename__ = 'menu_item'
#column definitions for the restaurant table
name = Column(String(80), nullable=False)
id = Column(Integer, primary_key=True)
course = Column(String(250))
description = Column(String(250))
price = Column(String(8))
restaurant_id = Column(Integer, ForeignKey('restaurant.id'))
restaurant = relationship(Restaurant)
#create an instance of create_engine class
#and point to the database to be used
engine = create_engine(
'sqlite:///restaurantmenu.db')
#that will soon be added into the database. makes
#the engine
Base.metadata.create_all(engine)
I can't figure out why i cannot add new restuarants
I know this was a long time ago, but I figured out your problem.
First, the enctype='multipart/form-data' in your do_GET function under the if self.path.endswith("/restaurants/new"): portion is missing a final single quote. Second, you misspelt 'multipart' in if ctype == 'multipart/form-data':. Hope that can help you or others.
As shteeven said, the problem was with the encryption type in the form.
As the quote was missed, the 'Content-type' changed to 'application/x-www-form-urlencoded' so in that case you should parse it different as it's a string.
In order to manage both enctype you can modify your do_POST as the following
def do_POST(self):
try:
if self.path.endswith("/restaurants/new"):
ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
print ctype
#check of content-type is form
if (ctype == 'multipart/form-data') or (ctype == 'application/x-www-form-urlencoded'):
#collect all fields from form, fields is a dictionary
if ctype == 'multipart/form-data':
fields = cgi.parse_multipart(self.rfile, pdict)
else:
content_length = self.headers.getheaders('Content-length')
length = int(content_length[0])
body = self.rfile.read(length)
fields = urlparse.parse_qs(body)
#extract the name of the restaurant from the form
messagecontent = fields.get('newRestaurant')
#create the new object
newRestaurantName = Restaurant(name = messagecontent[0])
session.add(newRestaurantName)
session.commit()
self.send_response(301)
self.send_header('Location','/restaurants')
self.end_headers()
return
Hope this extra information is useful for you!
I want to use TermRaider features with GATE. Could someone please post some sample code to load and use this resource in java class. I have tried with following but failed.
Gate.getCreoleRegister().registerDirectories(new URL("file:///D:/misc_workspace/gate-7.1-build4485-SRC/plugins/TermRaider"));
ProcessingResource termRaider = (ProcessingResource) Factory.
createResource("gate.termraider.TermRaiderEnglish",Factory.newFeatureMap());
Exception:
gate.termraider.TermRaiderEnglish cannot be cast to gate.ProcessingResource
Could anyone please suggest how should I proceed.
The TermRaider system isn't a single PR, it's a whole application (in fact a Groovy ScriptableController). The TermraiderEnglish Resource is just a hook to make that application appear in the "ready-made applications" menu of the GATE Developer GUI.
In embedded code you can load the application using the PersistenceManager
File termRaiderPlugin = new File(Gate.getPluginsHome(), "TermRaider");
File gappFile = new File(new File(termRaiderPlugin, "applications"),
"termraider-eng.gapp");
CorpusController trApp = (CorpusController)PersistenceManager.loadObjectFromFile(
gappFile);
When you run the application over a corpus, it creates new instances of three "termbank" LRs containing the information about the newly discovered terms. The vanilla application is really intended for GUI rather than embedded use so it doesn't store references to these new LRs anywhere useful - you'll have to interrogate the CreoleRegister to find them. You might prefer to make your own copy of the application and tweak the control script to store the termbank instances as (say) features on the Corpus, by adding something like
corpus.features.tfidfTermbank = termbank0
corpus.features.annotationTermbank = termbank1
corpus.features.hyponymyTermbank = termbank2
to the end of the control script. You could then access them in your Java code via corpus.getFeatures().get("tfidfTermbank") etc.
Since these Termbank classes are themselves part of the TermRaider plugin, you'll probably want to add gate-termraider.jar to your main application classpath rather than loading it via the GateClassLoader.
import gate.Corpus;
import gate.CorpusController;
import gate.Document;
import gate.Factory;
import gate.FeatureMap;
import gate.Gate;
import gate.termraider.bank.AbstractTermbank;
import gate.termraider.output.CsvGenerator;
import gate.util.GateException;
import gate.util.Out;
import gate.util.persistence.PersistenceManager;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.net.URLDecoder;
public class termraider {
public static void main(String[] args) throws IOException, GateException {
// initialise the GATE library
Out.prln("Initialising GATE...");
Gate.init();
// Initialize GATE
File gateHome = Gate.getGateHome();
Out.prln("...GATE initialised");
//Load TermRaider plugin
File termRaiderPlugin = new File(Gate.getPluginsHome(), "TermRaider");
File gappFile = new File(new File(termRaiderPlugin, "applications"),
"termraider-eng.gapp");
CorpusController trApp = (CorpusController)PersistenceManager.loadObjectFromFile(gappFile);
System.out.println("TermRaider loaded successfully!!!");
//Loading txt files from a folder path
Corpus corpus = (Corpus) Factory.createResource("gate.corpora.CorpusImpl");
//String dirname = "Desktop/Gate_corpus/About Us/New Folder";
String dirname = "Desktop/GermanHPFCompetition/termRaider";
File f1 = new File(dirname);
String s[] = f1.list();
for (int i=0; i < s.length; i++) {
String path = dirname + "/" + s[i];
path = URLDecoder.decode(path, "utf-8");
path = new File(path).getPath();
URL u=new URL("file:\\\\\\"+path);
FeatureMap params = Factory.newFeatureMap();
params.put("sourceUrl", u);
params.put("preserveOriginalContent", new Boolean(true));
params.put("collectRepositioningInfo", new Boolean(true));
//Out.prln("Creating doc for " + u);
Document doc = (Document)
Factory.createResource("gate.corpora.DocumentImpl", params);
corpus.add(doc);
} // for each file in the folder
//running TermRaider plugin with the corpus
trApp.init();
trApp.setCorpus(corpus);
trApp.execute();
Corpus output_corpus = (Corpus) Factory.createResource("gate.corpora.CorpusImpl");
output_corpus=trApp.getCorpus();
System.out.println("TermRaider executed successfully!!!");
//Creating csv files as output
AbstractTermbank tb1 = (AbstractTermbank) output_corpus.getFeatures().get("tfidfTermbank");
AbstractTermbank tb2 = (AbstractTermbank) output_corpus.getFeatures().get("hyponymyTermbank");
AbstractTermbank tb3 = (AbstractTermbank) output_corpus.getFeatures().get("annotationTermbank");
System.out.println(tb1);
System.out.println(tb2);
System.out.println(tb3);
CsvGenerator generator = new CsvGenerator();
File outputFile1 = new File("Desktop/GermanHPFCompetition/termRaider/tfidfTermbank.csv");
File outputFile2 = new File("Desktop/GermanHPFCompetition/termRaider/hyponymyTermbank.csv");
File outputFile3 = new File("Desktop/GermanHPFCompeti`enter code here`tion/termRaider/annotationTermbank.csv");
double threshold1 = 0;
double threshold2 = 0;
double threshold3 = 0;
generator.generateAndSaveCsv(tb1, threshold1, outputFile1);
generator.generateAndSaveCsv(tb2, threshold2, outputFile2);
generator.generateAndSaveCsv(tb3, threshold3, outputFile3);
System.out.println("CSV files created!!!");
}//end of main
}//end of class