Is there anyway to get datastore entity legacy urlsafe in dataflow?
I've tried some method but all failed...
Direct encode
base64.urlsafe_b64encode(key.SerializeToString()).strip(b'=')
=> not the same compared with appengine
import google.cloud.datastore.key
=> seems conflict with dataflow datastoreio?
Copy _app_engine_key_pb2.py from google.cloud.datastore and use it just like to_legacy_urlsafe() source code
elements = []
for part in key.path:
element_kwargs = {'type': part.kind}
if part.id:
element_kwargs['id'] = part.id
elif part.name:
element_kwargs['name'] = part.name
element = _app_engine_key_pb2.Path.Element(**element_kwargs)
elements.append(element)
legacy_path = _app_engine_key_pb2.Path(element=elements)
reference = _app_engine_key_pb2.Reference(
app=key.partition_id.project_id,
path=legacy_path,
name_space=key.partition_id.namespace_id
)
raw_bytes = reference.SerializeToString()
urlsafe = base64.urlsafe_b64encode(raw_bytes).strip(b'=')
=> still not the same compared with appengine...
The ndb code has:
urlsafe = base64.b64encode(self.reference().Encode())
return urlsafe.rstrip('=').replace('+', '-').replace('/', '_')
So you should get the same results with:
urlsafe = base64.b64encode(key.SerializeToString())
return urlsafe.rstrip('=').replace('+', '-').replace('/', '_')
Are you using ndb or the db library? The to_legacy_urlsafe() function you found seems to match the db library.
Related
I would like to know how I should change my code to prevent it from Injections:
import sqlite3
def search_in_database(column,searched_data):
con = sqlite3.connect('db.sqlite3')
cursor = con.cursor()
cursor.execute(f"""SELECT
id
FROM
My_library
WHERE
{column} LIKE '%{searched_data}%'
;""")
all = [i for i in cursor.fetchall()]
return all
I found code in web which gives an example of how to do it:
from sqlalchemy.sql import text
# Create a connection conn
stmt = text("""SELECT * FROM users
WHERE user = :username AND password = :password""")
conn.execute(stmt, prams={"username": "foo", "password": "bar"})
but In my HTML file I would like to give to user possibility to choose the:
Place where he wants to search in Titles, authors, published_dates,isbn, language...
and when he choose where He what to search then he types the query.
How to do it in this case, avoiding Injections?
My data base:
class My_library(db.Model):
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String(250))
authors = db.Column(db.String(100))
published_date = db.Column(db.Integer)
isbn_or_identifier = db.Column(db.String(15), unique=True)
page_count = db.Column(db.String(10000))
language = db.Column(db.String(3))
image_links = db.Column(db.String(500))
I also added validators:
from flask_wtf import FlaskForm
from wtforms import SubmitField,StringField
from wtforms.validators import ValidationError,DataRequired,Length, URL
from wtforms.fields.html5 import DateField,IntegerField,DateField,IntegerField, URLField
class AddRecValidators(FlaskForm):
title = StringField(label=('Title:'),validators=[DataRequired(), Length(min=1,max=50)])
authors = StringField(label=('Authors:'),validators=[Length(min=1,max=100)])
published_date = IntegerField(label=('Published date:'),validators=[Length(min=1,max=4)])
isbn_or_identifier = IntegerField(label=('ISBN:'),validators=[Length(min=1,max=15)])
page_count = IntegerField(label=('Page count:'),validators=[ Length(min=1,max=10000)])
language = StringField(label=('Language:'),validators=[ Length(min=1,max=3)])
image_links = URLField(label=('Image links:'))
submit = SubmitField(label=('Add to library'))
Thanks for help in advance :D
You can use sqlalchemy to build the query. For example:
q = My_library.__table__.select().where(
My_library.__table__.c.title == "The searched title"
)
but that's not exactly what you wanted. You can also address the columns by their names like this:
q = My_library.__table__.select().where(
My_library.__table__.c["title"] == "The searched title"
)
# or
q = My_library.__table__.select().where(
My_library.__table__.c["title"].like("%The searched title%")
)
Therefore you can do this:
q = My_library.__table__.select().where(
My_library.__table__.c[column].like(f"%{searched_data}%")
)
cursor.execute(q)
In case you only want the ID, you would do this:
q = sqlalchemy.select([My_library.__table__.c.id]).where(
My_library.__table__.c[column].like(f"%{searched_data}%")
)
# you can print(q) to see what it constructed
cursor.execute(q)
That was SQLAlchemy Query Language. You are using ORM. I suggest you read-up something about a session in flask first.
It is still possible to get to the column-name related attribute and I am not sure this is the most efficient way:
q = session.query(My_library.id).filter(
getattr(My_library, column).like(f"%{searched_data}%"),
)
The goal is simple: to join two firestore queries utilizing rxjs, rxfire, and the rnfirebase react native library.
I've read multiple tutorials 1, 2 on joining queries, but they all fail with different errors.
//Simple test for collectionData
import { collectionData } from 'rxfire/firestore';
this.myQuery = this.props.docRef.collection(`messages`).where('read', 'array-contains', this.props.me.uid)
collectionData(this.myQuery, 'id').subscribe(docs => console.log(docs))
//Fails with error: this._next is not a function.
Alternatively,
this.publicQuery = this.props.docRef.collection('messages').where('public', '==', true)
this.myQuery = this.props.docRef.collection(`messages`).where('read', 'array-contains', this.props.me.uid)
const myQuery$ = new Rx.Subject();
const publicQuery$ = new Rx.Subject();
this.myQuery.onSnapshot((querySnapshot) => {
myQuery$.next(querySnapshot.docs.map(d => d.data() ));
});
this.publicQuery.onSnapshot((querySnapshot) => {
publicQuery$.next(querySnapshot.docs.map(d => d.data() ));
});
const orQuery$ = combineLatest(this.myQuery, this.publicQuery).switchMap((docs) => {
var [one, two] = docs;
var combined = one.concat(two);
return Rx.Observable.of(combined);
})
orQuery$.subscribe((result) => {
console.log('>>>> ', result)
})
//TypeError: undefined is not a function (near ...switchMap)
How can I successfully join two firestore queries (OR)?
You're already very close to the solution. Let's go through the issues step-by-step.
First, it's not necessary to create a Subject just to transform your result from onSnapshot. Instead of this:
this.myQuery.onSnapshot((querySnapshot) => {
myQuery$.next(querySnapshot.docs.map(d => d.data()))
});
We can achieve the same using 'pipeable transformation operators':
const myQuery$ = this.myQuery.onSnapshot.pipe(
map(querySnapshot => querySnapshot.docs.map(d => d.data()))
);
The same goes for the other query:
const publicQuery$ = this.publicQuery.onSnapshot.pipe(
map(querySnapshot => querySnapshot.docs.map(d => d.data())
);
Second, to join those two queries, combineLatest is indeed the correct creation function.
However, your error might result from you using a newer RxJS version, that doesn't support 'fluent' operators (officially called "patch operators") anymore. They have been replaced by 'pipeable operators' from RxJS 6 onwards. As an example, myObs$.map(...) has become myObs$.pipe(map(...)). The tutorials probably use an older version of RxJS where the first is still possible.
Also, it shouldn't be necessary to use switchMap if the inner Observable is just an of operator. It is sufficient in this case to use the map operator, which will behave equally.
Using the new RxJS 6+ syntax together with map, the combination will look like this:
const orQuery$ = combineLatest(myQuery$, publicQuery$).pipe(
map(([one, two]) => one.concat(two))
)
The rest of your code should be correct.
Side Note: Keep in mind that the equivalent of your code in SQL is UNION (not JOIN). In order to JOIN programatically, you'd need to combine each object of result set A with each object of result set B and create a joined object for each pair. Such a function for a keyless OUTER JOIN would look like this (placed in your map pipe):
one.map(a =>
two.map(b => Object.assign({}, a, b)))
.reduce((p, c) => p.concat(c), [])
If you want to have a UNION with no duplicate objects, concat only those items from two that have no matching primary key in list one. This would be your mapping function:
one.concat(two.filter(twoItem => !one.some(oneItem => oneItem.id == twoItem.id)))
DEMO: A complete, working demo with the above code and a simulated FireStore can be found here:
https://stackblitz.com/edit/rxjs-mefynu?devtoolsheight=60
I've created a dictionary from an Uploaded file in Django.
This dictionary has a nested list of dictionaries:
file = {"name": "filename", "sections": [{"section_name": "string", "lines": [{line_number: 0, "line"; "data"}]}], "etc": "etc"}
The model represents the dictionaries depth too.
class Line(EmbeddedDocument):
line_number = IntField()
line = StringField()
definition = ReferenceField(Definition)
class Section(EmbeddedDocument):
section_name = StringField()
lines = EmbeddedDocumentListField(Line))
class File(Document):
name = StringField()
sections = EmbeddedDocumentListField(Section))
created_on = DateTimeField()
created_by = StringField()
modified_on = DateTimeField()
modified_by = StringField()
In the POST I have the following to chop the file up into the above Dict (the file is a simple text file):
file= {}
with open(os.path.join(path, filename + ".txt"), 'r') as temp_file:
filelines = temp_file.readlines()
sections = []
section = {}
lines = []
for i, l in enumerate(filelines):
if i == 0:
section["section_name"] = "Top"
elif '*' in l:
if l.index('*') == 0 and '*' not in lines[len(lines) - 2"line"]:
section["lines"] = lines
lines = []
sections.append(section)
section = dict()
section["section_name"] = filelines[i + 1][1:-2]
line = {"line_number": i + 1, "line": l}
lines.append(line)
section['lines'] = lines
sections.append(section)
file["name"] = filename
file["sections"] = sections
I will tidy this up eventually.
Once the dict has been made how do I serialise it using the serializer?
Is it possible to insert this into a serializer?
If not how can I get it all into the database with validation?
I've tried json.dumps() and JsonRequst() then putting them in data= for the serializer but get Unable to get repr for <class '....'>
I'm pretty new to Django and MongoDB so if you need more info I can provide :)
Thanks!
Update
Change the model's List Fields to EmbeddedDocumentListField as suggest in the answer.
Answered
Thanks to Boris' suggestion below it pointed me to an error I wasn't getting initially. I had a typo and passing the dict directly into FileSerializer(data=file) works like a charm! :)
James!
The easiest way to validate that your incoming JSONs adhere to the Mongoengine Documents schema that you've specified is to use DRF-Mongoengine's DocumentSerializer.
Basically, what you need to do is create a serializer
serializers.py
import rest_framework_mongoengine
class FileSerializer(rest_framework_mongoengine.DocumentSerializer):
class Meta:
fields = '__all__'
model = File
Then you need a view or viewset that makes use of this Serializer to respond to GET/POST/PUT/DELETE requests.
views.py
from rest_framework_mongoengine import viewsets
class FileViewSet(viewsets.ModelViewSet):
lookup_field = 'id'
serializer_class = FileSerializer
def get_queryset(self):
return File.objects.all()
and register this viewset with a router
urls.py
from rest_framework import routers
# this is DRF router for REST API viewsets
router = routers.DefaultRouter()
# register REST API endpoints with DRF router
router.register(r'file', FileViewSet, r"file")
I'd also recommend using EmbeddedDocumentListField instead of ListField(EmbeddedDocumentField(Section)) - it has additional methods.
I researched this and all I can find is a suggestion to turn on .Trace = true like this:
db1 = DependencyService.Get<ISQLite>().GetConnection();
db1.Trace = true;
I also tried this:
db2.Trace = true;
var categories = db2.Query<Category>("SELECT * FROM Category ORDER BY Name").ToList();
Debug.WriteLine("xxxx");
Well I did this and then restarted the application. When I view the Application output I just see information on threads started and the xxxx but don't see any SQL trace information.
Can anyone give me advice on this. Thanks
You need to set Trace and Tracer (action) properties on your SQLiteConnection to print queries to output:
db.Tracer = new Action<string>(q => Debug.WriteLine(q));
db.Trace = true;
Look in the Application Output window for lines that begin Executing
Example Output after setting Trace to true:
Executing: create table if not exists "Valuation"(
"Id" integer primary key autoincrement not null ,
"StockId" integer ,
"Time" datetime ,
"Price" float )
Executing Query: pragma table_info("Valuation")
Executing: create index if not exists "Valuation_StockId" on "Valuation"("StockId")
Executing: insert into "Stock"("Symbol") values (?)
Executing Query: select * from "Stock" where ("Symbol" like (? || '%'))
0: A
Ref: https://github.com/praeclarum/sqlite-net/blob/38a5ae07c886d6f62cecd8fdeb8910d9b5a77546/src/SQLite.cs
The SQLite PCL uses Debug.WriteLine which means that the logs are only included in Debug builds of the PCL.
Remove your nuget reference to the sqlite.net PCL (leave the native reference), and instead add SQLite.cs as a class to your project, and execute a debug build, with the Trace flag set, and you'll see the tracing.
I didn't have to do anything special other than include the SQLite.cs file in my Xamarin iOS project for this to work:
using (var conn = new SQLite.SQLiteConnection("mydb.sqlite") { Trace = true }) {
var rows = conn.Table<PodcastMetadata>().Where(row => row.DurationMinutes < 10).Select(row => new { row.Title });
foreach (var row in rows) {
Debug.WriteLine(row);
}
}
Output:
Executing Query: select * from "PodcastMetadata" where ("DurationMinutes" < ?)
0: 10
Trying to get working ScalikeJDBC and SQLite. Have a simple code based on provided examples:
import scalikejdbc._, SQLInterpolation._
object Test extends App {
Class.forName("org.sqlite.JDBC")
ConnectionPool.singleton("jdbc:sqlite:test.db", null, null)
implicit val session = AutoSession
println(sql"""SELECT * FROM kv WHERE key == 'seq' LIMIT 1""".map(identity).single().apply()))
}
It fails with exception:
Exception in thread "main" java.sql.SQLException: Cannot change read-only flag after establishing a connection. Use SQLiteConfig#setReadOnly and QLiteConfig.createConnection().
at org.sqlite.SQLiteConnection.setReadOnly(SQLiteConnection.java:447)
at org.apache.commons.dbcp.DelegatingConnection.setReadOnly(DelegatingConnection.java:377)
at org.apache.commons.dbcp.PoolingDataSource$PoolGuardConnectionWrapper.setReadOnly(PoolingDataSource.java:338)
at scalikejdbc.DBConnection$class.readOnlySession(DB.scala:138)
at scalikejdbc.DB.readOnlySession(DB.scala:498)
...
I've tried both scalikejdbc 1.7 and 2.0, error remains. As sqlite driver I use "org.xerial" % "sqlite-jdbc" % "3.7.+".
What can I do to fix the error?
The following will create two separate connections, one for read-only operations and the other for writes.
ConnectionPool.add("mydb", s"jdbc:sqlite:${db.getAbsolutePath}", "", "")
ConnectionPool.add(
"mydb_ro", {
val conf = new SQLiteConfig()
conf.setReadOnly(true)
val source = new SQLiteDataSource(conf)
source.setUrl(s"jdbc:sqlite:${db.getAbsolutePath}")
new DataSourceConnectionPool(source)
}
)
I found that the reason is that you're using "org.xerial" % "sqlite-jdbc" % "3.7.15-M1". This version looks still unstable.
Use "3.7.2" as same as #kawty.
Building on #Synesso's answer, I expanded slightly to be able to get config value from config files and to set connection settings:
import scalikejdbc._
import scalikejdbc.config.TypesafeConfigReader
case class SqlLiteDataSourceConnectionPool(source: DataSource,
override val settings: ConnectionPoolSettings)
extends DataSourceConnectionPool(source)
// read settings for 'default' database
val cpSettings = TypesafeConfigReader.readConnectionPoolSettings()
val JDBCSettings(url, user, password, driver) = TypesafeConfigReader.readJDBCSettings()
// use those to create two connection pools
ConnectionPool.add("db", url, user, password, cpSettings)
ConnectionPool.add(
"db_ro", {
val conf = new SQLiteConfig()
conf.setReadOnly(true)
val source = new SQLiteDataSource(conf)
source.setUrl(url)
SqlLiteDataSourceConnectionPool(source, cpSettings)
}
)
// example using 'NamedDB'
val name: Option[String] = NamedDB("db_ro") readOnly { implicit session =>
sql"select name from users where id = $id".map(rs => rs.string("name")).single.apply()
}
This worked for me with org.xerial/sqlite-jdbc 3.28.0:
String path = ...
SQLiteConfig config = new SQLiteConfig();
config.setReadOnly(true);
return DriverManager.getConnection("jdbc:sqlite:" + path, config.toProperties());
Interestingly, I wrote a different solution on the issue on the xerial repo:
PoolProperties props = new PoolProperties();
props.setDriverClassName("org.sqlite.JDBC");
props.setUrl("jdbc:sqlite:...");
Properties extraProps = new Properties();
extraProps.setProperty("open_mode", SQLiteOpenMode.READONLY.flag + "");
props.setDbProperties(extraProps);
// This line can be left in or removed; it no longer causes a problem
// as long as the open_mode code is present.
props.setDefaultReadOnly(true);
return new DataSource(props);
I don't recall why I needed the second, and was then able to simplify it back to the first one. But if the first doesn't work, you might try the second. It uses a SQLite-specific open_mode flag that then makes it safe (but unnecessary) to use the setDefaultReadOnly call.