Trouble reading sqlite3 database columns of type blob with sql.js - sqlite

So i am using the sql.js library i.e. the port of sqlite in javascript which can be found here https://github.com/kripken/sql.js.
This is my code to open and read the database that comes from a flat file store locally.
First the file a local file is selected via this HTML
<input type="file" id="input" onchange="handleFiles(this.files)">
The js code behind the scenes is as follows,
function handleFiles(files) {
var file = files[0];
var reader = new FileReader();
reader.readAsBinaryString(file);
openDbOnFileLoad(reader);
function openDbOnFileLoad(reader){
setTimeout(function () {
if(reader.readyState == reader.DONE) {
//console.log(reader.result);
db = SQL.open(bin2Array(reader.result));
execute("SELECT * FROM table");
} else {
//console.log("Waiting for loading...");
openDbOnFileLoad(reader);
}
}, 500);
}
}
function execute(commands) {
commands = commands.replace(/\n/g, '; ');
try {
var data = db.exec(commands);
console.log(data);
} catch(e) {
console.log(e);
}
}
function bin2Array(bin) {
'use strict';
var i, size = bin.length, ary = [];
for (i = 0; i < size; i++) {
ary.push(bin.charCodeAt(i) & 0xFF);
}
return ary;
}
Now this works and i can access all the columns and values in the database, however there is one column which is of type blob and that just shows up as empty. Any ideas of how i can access the contents of this blob?
The correct answer!
So what I was trying to ask in this question is simply how to read the contents of a column of type blob using sql.js. The correct answer is to specify the column names in the question and for the column that contains data of type blob, get its contents using the hex function i.e. select column1,hex(column2) from table. It was by no means a question about the most efficient way of doing this. I have also written a blog post about this.

Here is a slightly modified copy of the function responsible for initializing my sqlite database:
sqlite.prototype._initQueryDb = function(file, callback) {
self = this;
var reader = new FileReader();
// Fires when the file blob is done loading to memory.
reader.onload = function(event) {
var arrayBuffer = event.target.result,
eightBitArray = new Uint8Array(arrayBuffer),
database = SQL.open(eightBitArray);
self._queryDb = database;
// Trigger the callback to the calling function
callback();
}
// Start reading the file blob.
reader.readAsArrayBuffer(file);
}
In this case, file is a local sqlite database handle that I get from an HTML input element. I specify a function to call when a change event happens to that input and get the blob from the resulting event.target.files[0] object.
For the sake of brevity on my part I left some things out but I can throw together a smaller and more simplified example if you are still struggling.

The answer is: with kripken's sql.js, that you mentioned above you can't. At least as of today (may 2014). The original author doesn't maintain sql.js anymore.
However, I'm the author of a fork of sql.js, that is available here: https://github.com/lovasoa/sql.js .
This fork brings several improvements, including support for prepared statements, in which, contrarily to the original version, values are handled in their natural javascript type, and not only as strings.
With this version, you can handle BLOBs (both for reading and writing), they appear as Uint8Arrays (that you can for instance convert to object URL to display contents to your users).
Here is an example of how to read blob data from a database:
var db = new SQL.Database(eightBitArray); // eightBitArray can be an Uint8Array
var stmt = db.prepare("SELECT blob_column FROM your_table");
while (stmt.step()) { // Executed once for every row of result
var my_blob = stmt.get()[0]; // Get the first column of result
//my_blob is now an Uint8Array, do whatever you want with it
}
db.close(); // Free the memory used by the database
You can see the full documentation here: http://lovasoa.github.io/sql.js/documentation/

Related

Auto Sync google sheets to firebase without button

I used a tutorial to help me sync my sheets to firebase with the use of a SYNC button that activates the script. The SYNC button currently sits just in the middle of the spreadsheet. I want to sync the data from sheets automatically to firebase when there are changes made.
function getFirebaseUrl(jsonPath) {
return (
'https://no-excusas.firebaseio.com/' +
jsonPath +
'.json?auth=' +
secret
)
}
function syncMasterSheet(sheetHeaders, sheetData) {
/*
We make a PUT (update) request,
and send a JSON payload
More info on the REST API here : https://firebase.google.com/docs/database/rest/start
*/
const outputData = [];
for(i = 0; i < sheetData.length; i++) {
var row = sheetData[i];
var newRow = {};
for(j = 0; j < row.length; j++) {
newRow[sheetHeaders[j]] = row[j];
}
outputData.push(newRow);
}
var options = {
method: 'put',
contentType: 'application/json',
payload: JSON.stringify(outputData)
}
var fireBaseUrl = getFirebaseUrl("UsersSheets")
UrlFetchApp.fetch(fireBaseUrl, options)
}
function startSync() {
//Get the currently active sheet
var sheet = SpreadsheetApp.getActiveSheet()
//Get the number of rows and columns which contain some content
var [rows, columns] = [sheet.getLastRow(), sheet.getLastColumn()]
// Get the data contained in those rows and columns as a 2 dimensional array.
// Get the headers in a separate array.
var headers = sheet.getRange(1, 1, 1, columns).getValues()[0]; // [0] to unwrap the
outer array
var data = sheet.getRange(2, 1, rows - 1, columns).getValues(); // skipping the header
row means we need to reduce rows by 1.
//Use the syncMasterSheet function defined before to push this data to the "masterSheet"
key in the firebase database
syncMasterSheet(headers, data)
}
Normally, it would be ok to just define an onEdit function in your code, like this:
function onEdit(event) {
startSync();
}
However, because you are making external requests via UrlFetchApp.fetch(), this will fail with an error about not having the https://www.googleapis.com/auth/script.external_request permission (gobs more detail about trigger authorization here).
Instead, you need to manually create an installable trigger
This is reasonably straightforward. In the edit menu for your code, go to your project's triggers:
Then, select "add a trigger" and create the on edit trigger, like so:
You should think about if you really want this running on every edit as the requests could be quite large (as it syncs the entire sheet) and run frequently (as you edit), however.
When you make a change to a spreadsheet, its onEdit event fires. So that's where you'd trigger that save with something like this:
function onEdit(event) {
startSync();
}
But since onEdit fires for each edit, this may end up saving a lot more than really necessary. So you may want to debounce to only save after some inactivity.
Something like this:
var timer;
function onEdit(event) {
// if we're counting down, stop the timer
if (timer) clearTimeout(timer);
// starting syncing after 2 seconds
timer = setTimeout(function() {
startSync();
}, 2000);
}

Deleting data from Drive Tables and automatically re-importing new data

I need help with trying to understand how to delete all data from a table and then try to automatically import a new sheet with data into the newly cleared down table.
I'm currently trying the unload() method client side but that doesn't seem to cleardown my tables
function ClearDown(){
app.datasources.P11d.unload(function(){});
console.log('Finish Delete');
}
I've also tried to create a server side function, which also doesn't appear to work
function ClearTable(){
var records = app.models.P11d.newQuery();
// records.run();
console.log('Server Function Ran');
app.deleteRecords(records.run());
}
This is ran from a client side function:
function Delete(){
google.script.run.withSuccessHandler(function(result){
}).ClearTable();
console.log('Function Ran');
}
Again this is all to no avail
With the import side I've tried to do the below:-
Client Side:
function ImportData(){
console.log('Begin');
var ss = SpreadsheetApp.openById('SHEET ID');
var values = ss.getSheetByName('P11d').getDataRange().getValues();
var ssData = [];
// app.datasources.P11d.unload(function(){});
for (var i = 0; i<values.length; i++){
var newRecord = app.models.P11d.newRecord();
// add all fields to the new record
newRecord.Reg_Number = values[i][0];
newRecord.Reg_Date = values[i][1];
newRecord.Model_Description = values[i][2];
newRecord.P11d_Value = values[i][3];
newRecord.EngineSize = values[i][4];
newRecord.Fuel = values[i][5];
newRecord.CO2 = values[i][6];
newRecord.SIPP = values[i][7];
newRecord.GTA_Code = values[i][8];
newRecord.Type = values[i][9];
ssData.push(newRecord);
// console.log(newRecord.MODEL_FIELD);
}
console.log('Finished');
// return the array of the model.newRecord objects that would be consumed by the Model query.
}
Please can someone help with this, at the moment the way the data is sent over to me adding new stuff into the Drive Table is causing many duplicates.
Thanks in advance,
You can delete all records, import, and read from a spreadsheet using the AMU Library
Copy and paste the server and client scripts into your app.
I'm sure that will make it much easier!
To delete all the data in a model using this:
Button onClick:
google.script.run.AMU.deleteAllData('ModelName');
The correct way to delete records on the server is:
app.models.MODEL_NAME.deleteRecords(key_array);
datasource.unload() simply unloads the widget on the client. It does not affect the database records.
A better way to write your records query on the server is:
var query = app.models.MODEL_NAME.newQuery();
query.filters.your_filter_here;
var records = query.run();
Note that you cannot return a single record or an array of records from anything but a calculated model function without using a function posted here. (You can return a single field of a record using stringify for any json data.)
I am currently working on a solution to create datasource independent tables needed in App Maker.
For the delete function on the server try to change your code just a little bit, this function at least used to work for me, however I have not needed to use it in some time.
function ClearTable(){
var records = app.models.P11d.newQuery().run();
console.log('Server Function Ran');
app.deleteRecords(records);
}

AngularFire extending the service issue

I've been looking at the documentation for Synchronized Arrays https://www.firebase.com/docs/web/libraries/angular/api.html#angularfire-extending-the-services and https://www.firebase.com/docs/web/libraries/angular/guide/extending-services.html#section-firebasearray
I'm using Firebase version 2.2.7 and AngularFire version 1.1.2
Using the code below, I'm having trouble recognizing $$removed events.
.factory("ExtendedCourseList", ["$firebaseArray", function($firebaseArray) {
// create a new service based on $firebaseArray
var ExtendedCourseList= $firebaseArray.$extend({
$$added: function(dataSnapshot, prevChild){
var course = dataSnapshot.val();
var course_key = dataSnapshot.key();
console.log("new course");
return course;
},
$$removed: function(snap){
console.log("removed");
return true;
}
});
return function(listRef) {
return new ExtendedCourseList(listRef);
}
}])
.factory("CompanyRefObj", function(CompanyRef) {
//CompanyRef is a constant containing the url string
var ref = new Firebase(CompanyRef);
return ref;
})
.factory('CourseList', function (localstorage,$rootScope,ExtendedCourseList,CompanyRefObj) {
var companyID = localstorage.get("company");
$rootScope.courseList = ExtendedCourseList(CompanyRefObj.child(companyID).child("courses"));
)
If I run this code, only the $$added events will be triggered. To simulate the remove events I use the web-interface at Firebase to display data, where I press the remove button and accept the data being deleted permanently.
Additionally, if I delete the $$removed function, the extended service still won't synchronize when a record is deleted.
If I modify my code to use the $firebaseArray instead of extending the service (as seen above) both add and remove events will be recognized.
.factory('CourseList', function (localstorage,$rootScope,$firebaseArray,CompanyRefObj) {
var companyID = localstorage.get("company");
$rootScope.courseList = $firebaseArray(CompanyRefObj.child(companyID).child("courses"));
)
Finally, are there any bad practices I've missed that can cause some of the extended functions to not work?
Solved
$$added: function(dataSnapshot, prevChild){
var course = dataSnapshot.val();
var course_key = dataSnapshot.key();
//Modified below
course.$id = course_key;
//End of modification
console.log("new course");
return course;
}
After posting about the issue at firebase/angularfire github I received an answer that solved my issue. When $$added got overridden by the code provided, the $firebaseArray also lost its internal record $id.
Adding this line of code: course.$id = course_key; before returning the course, made AngularFire recognize when the record was removed from the server.

angularFire startAt querying and binding deletes new data

The application shows work-shifts for certain time-period. firebaseConn.getShifts is the API-function to get the shiftData for the given time period.
versions:
firebase: 2.0.6
angularFire: 0.9.0 (confirmed with 0.8.2 also)
This is my firebase schema:
And this is the code:
.factory('watchers', function(bunch-of-dependencies) {
var unbindShifts = function() {};
var inited = false;
var shifts = {};
... some irrelevant code in between ...
function initShifts() {
unbindShifts();
shifts.object = firebaseConn.getShifts( false, from, to, $scope );
$scope.shifts = shifts.object;
shifts.object.$bindTo($scope, "shifts").then(function(unbind) {
unbindShifts = unbind;
});
}
The firebase-queries (that have worked fine before adding the unbind / bind and possibly time-based querying might cause issues too):
firebaseConn.getShifts = function(asArray, from, to, scope) {
return cacheRequest(FBURL + "shifts", asArray, [from, to]);
};
function cacheRequest(url, asArray, limits) {
var type = asArray ? "array" : "object";
var startAt = limits ? limits[0] : undefined;
var endAt = limits ? limits[1] : undefined;
var retObj, FBRef;
cached[url] = cached[url] || {};
/* If there are limits-parameters we don't cache at all atm. Since those queries should be checked differently than static urls */
if(!limits && cached[url][type]) {
FBRef = cached[url][type];
} else {
FBRef = cached[url][type] = createFBRef(url, startAt, endAt);
}
if(asArray) {
retObj = FBRef.$asArray();
} else {
retObj = FBRef.$asObject();
}
return retObj;
}
function createFBRef(resourceURL, startAt, endAt) {
var modifiedObject = $firebase( createRef( resourceURL ).orderByKey().startAt(startAt).endAt(endAt) );
return modifiedObject;
}
function createRef(resourceURL) {
return new Firebase( resourceURL );
}
Now I have located the problem to be with the query limiting. If the from and to Dates are undefined, this works without problems. But I need to be able to limit the amount of data, since loading many years of workshift-data, to show a weeks time, won't be good :).
The actual problem is not displaying and fetching the data, everything works fine, it's related to the times and re-binding.
If I do any changes to e.g. "20150115"-table. For example I add another "groups"-child there. When i unbind and rebind, the whole "20150115"-table gets deleted and this holds true only to the latest changes. If I add multiple child to different dates e.g. "20150113", "20150114", "20150115" and the latest change is in "20150115" and then I unbind + re-bind another time from firebase, all the other root-paths will stay as they are, but the latest change in "20150115" will make the whole tree deleted.
I hope I make myself clear, so for safety I try to explain it again in simpler way.
- Changes to 1. "20150113", 2. "20150114", 3. "20150115" through the app.
- Changing timeline from UI causes: unbind + re-bind
- As a side-effect the whole "20150114" tree gets deleted.
The problem is somehow related to advanced querying with orderByKey().startAt(startAt).endAt(endAt) and binding.
Also for additional info. The data which is added through the UI gets added to the firebase database, but when the re-binding happens, the data is deleted from the database. Specifically on rebind, unbinding causes no issues, if I delay rebinding with timeout.
EDIT:
I have found the source of the actual issue. After the new binding is in place and everything seems to be in order, there is an angular watch event that kicks in. The event tries to save the last change user made before re-binding.
So if I have and active timeline for december (20141201 - 20141230) and I change "20141225"-data. Then change the timeline to 20150101 - 20150130, causing unbind and rebind (or manually fetching new data). There will be an event, after the binding has been done and everything seems to be in order, trying to save 20141225 data to either the new timeline (20150101 - 20150130) or the old one, not sure which one. This causes the firebase to actually delete the whole 20141225-tree, instead of saving the data.
The new data makes it into your Firebase fine, which you can see by either checking your Firebase dashboard or by running a quick snippet like this in your browser's dev console:
new Firebase("https://firebaseurl").once('value', function(s) { console.log(s.val()); })
The data even makes it back into your application. The only problem is that Angular doesn't know that new data has arrived, so it doesn't update the view with the new data.
Normally AngularFire's $asObject and $asArray methods take care of notifying AngularJS when new data arrives from Firebase. But since you are constantly creating new queries, you'll have to take care of that yourself.
There are a few ways to signal the new data to AngularJS and I'm definitely not an expert on which one is best. But if you add $scope.$apply(); to your setDays function it works:
function setDays(ref) {
var FBRange = setFBRange(ref, from, to);
var days;
unbindDays();
days = $firebase(FBRange).$asObject();
$scope.days = days;
days.$bindTo($scope, "days").then(function(unbind) {
unbindDays = unbind;
// As a result of the new binding entry gets mysteriously deleted from firebase
});
$scope.$apply(); // Tell AngularJS about the new data, so that it updates the view
function setFBRange(ref, from, to) {
return ref.orderByKey().startAt(""+from).endAt(from + to + "");
}
}
Updated Plunkr with this change (and some others to help in debugging): http://plnkr.co/edit/YZtkzUNtjQUCcw4xb2mj?p=preview

Capture photo with camera and store in DB Phonegap

I am trying to build an app using Phonegap (Cordova 2.2). I am fairly new to code and js in particular so please be patient if some of my questions seem daft. My question has a few parts:
I want the user to be able to enter name and image (using the device camera) to create their profile. I have tested on the emulator and so far they can enter their name and get take a photo with camera. These both display in the emulator on completion. However, obviously I want to save that detail, so, I have created a DB (from reading, local storage is limited in size).
1. Is the database done correctly? I am getting neither error nor success alerts.
2. I am struggling with the logic of how I pass the information into the database. I'm guessing that I call the populate_UsersDB() function on a click event and need to write some sort of INSERT INTO userProfiles VALUE something. Also, can I grab the Imagedata directly from the onPhotoFileSuccess() function and send it into the database from there.
3. If I don't want to have a done button (rather have a 'Tap here to take picture' placeholder), how do I test that the action is complete and send the info to the db. From reading, I think I can use onChange(), but not sure. Also, guessing I have to use post or get somewhere?
Sorry, I know there is a lot of questions within this question. Any advice or support is greatly appreciated. Thanks. Here's the code.
//Use the device camera to capture an image of the user
var pictureSource; // picture source
var destinationType; // sets the format of returned value
// Wait for PhoneGap to connect with the device
document.addEventListener("deviceready",onDeviceReady,false);
// PhoneGap is ready to be used!
function onDeviceReady() {
pictureSource=navigator.camera.PictureSourceType;
destinationType=navigator.camera.DestinationType;
}
// Called when a photo is successfully retrieved
function onPhotoDataSuccess(imageData) {
// Get image handle
var smallImage = document.getElementById('smallImage');
// Unhide image elements
//
smallImage.style.display = 'block';
// Show the captured photo
// The inline CSS rules are used to resize the image
//
smallImage.src = "data:image/jpeg;base64," + imageData;
}
// Called when a photo is successfully retrieved
function onPhotoFileSuccess(imageData) {
// Get image handle
console.log(JSON.stringify(imageData));
// Get image handle
//
var smallImage = document.getElementById('smallImage');
// Unhide image elements
//
smallImage.style.display = 'block';
// Show the captured photo
// The inline CSS rules are used to resize the image
//
smallImage.src = imageData;
}
// A button will call this function
function capturePhotoWithData() {
// Take picture using device camera and retrieve image as base64-encoded string
navigator.camera.getPicture(onPhotoDataSuccess, onFail, { quality:50, destinationType:Camera.DestinationType.DATA_URL });
}
function getPhoto(source) {
// Retrieve image file location from specified source
navigator.camera.getPicture(onPhotoURISuccess, onFail, { quality: 50,
destinationType: destinationType.FILE_URI,
sourceType: source });
}
// Called if something goes wrong.
function onFail(message) {
alert('Failed because: ' + message);
}
//CREATE THE DATABASE
document.addEventListener("deviceready", onDeviceReady, false);
var db = window.openDatabase("Users_DB", "1.0", "User Profiles DB", 200000); //will create database or open it
//function will be called when device ready
function onDeviceReady(){
db.transaction(populateUsers_DB, errorCB, successCB);
}
//create table
function populateUsers_DB(tx) {
tx.executeSql('CREATE TABLE IF NOT EXISTS userProfiles (id INTEGER PRIMARY KEY AUTOINCREMENT, Name TEXT NOT NULL, Image BLOB)');
tx.executeSql('INSERT INTO userProfiles(Name) VALUE ("")');
}
//function will be called when an error occurred
function errorCB(err) {
alert("Error processing SQL: "+err.code);
}
//function will be called when process succeed
function successCB() {
alert("success!");
db.transaction(queryDB,errorCB);
}
//select all from userProfiles
function queryDB(tx){
tx.executeSql('SELECT * FROM userProfiles',[],querySuccess,errorCB);
}
function querySuccess(tx,result){
$('#userList').empty();
$.each(result.rows,function(index){
var row = result.rows.item(index);
$('#userList').append('<li><h3 class="ui-li-heading">'+row['Image']+'</h3><p class="ui-li-desc">Name '+row['Name']+'</p></li>');
});
$('#userList').listview();
}

Resources