First commit

framework/sync/prepareSyncServer.js

@@ -0,0 +1,17 @@
+/*
+
+	Copyright (c) 2020, 2023, The Unified Company.
+
+	This code is part of Unify.
+
+	This program is free software; you can redistribute it and/or modify
+	it under the terms of the ESA Software Community License - Strong Copyleft,
+
+	https://unifyjs.org 
+
+*/
+
+import fse					from "fs-extra";
+
+fse.copySync("./framework/db/index.sqlite", "./framework/db/syncServer.sqlite");
+

framework/sync/switchDownloadedDatabaseFile.js

@@ -0,0 +1,19 @@
+/*
+
+	Copyright (c) 2020, 2023, The Unified Company.
+
+	This code is part of Unify.
+
+	This program is free software; you can redistribute it and/or modify
+	it under the terms of the ESA Software Community License - Strong Copyleft,
+
+	https://unifyjs.org 
+
+*/
+
+import fse					from "fs-extra";
+
+
+fse.copySync("./framework/db/index.sqlite", "./framework/db/backup"+Math.floor(Math.random() * 100)+".sqlite");
+
+fse.copySync("./framework/db/uploaded_db.sqlite", "./framework/db/index.sqlite");

framework/sync/syncClient.js.off

@@ -0,0 +1,492 @@
+/*
+
+	Copyright (c) 2020, 2023, The Unified Company.
+
+	This code is part of Unify.
+
+	This program is free software; you can redistribute it and/or modify
+	it under the terms of the ESA Software Community License - Strong Copyleft,
+
+	https://unifyjs.org 
+
+*/
+
+import WebSocket, {WebSocketServer} 	from 'ws';
+
+import database                     	from '../server/database.js';
+
+import querySQL 											from '../unify/querySQL.js';
+
+import readline 											from 'readline';
+
+import socketMessage 									from '../unify/socketMessage.js';
+
+import fs 														from 'fs';
+
+var clusterClientID = 0;
+
+
+//database.createTable( "sync_client" );
+
+database.addColumn( "original_id", "INTEGER", "sync_client" );
+
+database.addColumn( "server_id", "INTEGER", "sync_client" );
+
+database.addColumn( "queryString", "TEXT", "sync_client" );
+
+database.addColumn( "columnValues", "TEXT", "sync_client" );
+
+database.addColumn( "operation", "TEXT", "sync_client" );
+
+database.addColumn( "clusterClientID", "INTEGER", "sync_client" );
+
+database.addColumn( "synced", "BOOLEAN", "sync_client" );
+
+
+
+
+export default class syncClient{
+
+	clusterClientID				= 5;
+
+	syncServerAddress 		= "ws://localhost:4422";
+
+	mode 									= "sync";
+
+	constructor( mode ) {
+
+		if( mode ) {
+
+			this.mode = mode;
+
+		}
+
+		this.connect();
+
+	
+
+	}
+
+	emptyTable() {
+
+		database.database.prepare( "DELETE from sync_client" ).run()
+
+	}
+
+	hearthBeat() {
+
+		//if( this.socket.readyState != 0 )
+		
+		//console.log("perform hearthBeat");
+
+		var message 									= new socketMessage();
+
+		message.type									= "add_sqlite_operation";
+
+		message.unsynced 							= this.getUnsynced();
+
+		message.clusterClientID				= this.clusterClientID;
+
+		message.lastID 								= this.getLastID();
+
+		//console.log("getUnsynced", this.getUnsynced());
+
+		var unsynced 	= this.getUnsynced()
+
+		if(unsynced.length != 0) {
+
+			console.log( unsynced );
+
+			console.log( "unsynced", unsynced.length );
+			
+		}
+
+
+
+		if( this.socket ) {
+
+			this.socket.send( JSON.stringify( message ) );
+
+		}
+
+
+		var that  = this;
+
+	    setTimeout(function() {
+
+	     	that.hearthBeat();
+
+	    }, 1000);
+
+
+
+	}
+
+	async connect(){
+
+		this.socket = new WebSocket( this.syncServerAddress, {
+
+			perMessageDeflate: false
+
+		});
+
+		var syncClient 	= this;
+
+		this.socket.on('error', function open( error ) {
+
+			syncClient.onError();
+
+		});
+
+		this.socket.on('close', function open( error ) {
+
+			syncClient.onClose();
+			
+		});
+
+		this.socket.on('open', function open( socket ) {
+
+			syncClient.onOpen();
+
+		});
+
+		this.socket.on('message', function message( data ) {
+
+			syncClient.onMessage( data );
+
+		});
+
+	}
+
+	getLastID() {
+
+		var query = new querySQL( );
+		
+		query.type	= "select";
+
+		query.table	= "sync_client";
+
+		query.order	= ["server_id Desc LIMIT 1"];
+
+		query.find( "clusterClientID", this.clusterClientID , "!=" );
+
+		query.sync = false;
+
+
+
+		var result = database.query( query ); 
+
+		if( result.length == 0 ) {
+
+			return 0;
+
+		}
+
+		return result[0].server_id;
+
+	}
+
+	getUnsynced() {
+
+		var query = new querySQL( );
+		
+		query.type	= "select";
+		query.table	= "sync_client";
+		query.order	= ["id"];// Desc LIMIT 1
+
+		query.find( "clusterClientID", this.clusterClientID , "=" );
+
+		query.find( "synced", false );
+
+		query.sync	= false;
+
+		var result = database.query( query ); 
+
+		return result;
+
+	}
+
+	sendMessage( queryString, inputQuery, original_id ) {
+
+		//   !!! console.log("syncClient: send message is performed");
+		
+		if( !original_id ) {
+
+			return true;
+
+		}
+
+		var query = new querySQL( );
+
+
+		query.type  = "insert";
+
+		query.table = "sync_client";
+
+
+		query.setValue( "original_id", original_id );
+
+		query.setValue( "queryString", queryString );
+
+		query.setValue( "columnValues", JSON.stringify( inputQuery.getValues() ) ); 
+
+		query.setValue( "clusterClientID", this.clusterClientID ); 
+
+		query.setValue( "operation", inputQuery.type ); 
+
+		query.setValue( "synced", false ); 
+
+
+		query.sync	= false;
+
+
+		var result = database.query( query ); 
+
+	}
+
+
+
+	onError() {
+
+
+
+	}
+
+	onClose() {
+
+	  //console.log('Socket is closed. Reconnect will be attempted in 1 second.', this.mode);
+
+	  switch( this.mode ) {
+
+			case "download_database":
+
+				return true;
+
+			break;
+
+			default:
+
+				var that = this;
+
+				setTimeout(function() {
+
+				
+				that.connect();
+
+				}, 3000);
+
+		}
+
+	}
+
+	onOpen() {
+
+		switch( this.mode ) {
+
+			case "sync":
+
+				this.hearthBeat();
+
+			break;
+
+			case "download_database":
+
+				var message 									= new socketMessage();
+
+				message.type									= "download_database";
+
+				console.log("Downloading most recent database from Sync server, Please wait...");
+
+				this.socket.send( JSON.stringify( message ) );
+
+			break;
+
+
+		}
+			
+	}
+
+	onMessage( data ) {
+
+		var parseData = JSON.parse( data );
+
+		
+
+		switch( parseData.name ) {
+
+			case "connected":
+
+				//clusterClientID = parseData.clientID;
+
+			break;
+
+			case "upload_database":
+
+				//console.log("binary", parseData.binary);
+
+				console.log("Recieved most recent binary of database, Now writing to disk...");
+
+				fs.writeFileSync( "./framework/db/uploaded_db.sqlite", Buffer.from( parseData.binary.data )  );
+
+				this.socket.close()
+
+			break;
+
+			case "updateSyncClient":
+
+				// update synced rows
+				var synced = parseData.synced;
+
+				for (var i = 0; i < synced.length; i++) {
+
+					var row 		= synced[i];
+
+					var query 		= new querySQL( );
+
+
+					query.type  = "update";
+
+					query.table = "sync_client";
+
+					query.setValue( "synced", true );
+
+					//query.setValue( "server_id", row.server_id );
+
+
+					query.find( "id", row.id, "=" );
+
+
+					query.sync = false;
+
+
+					database.query( query );
+
+				}
+
+			//	console.log("set synced of previouse added rows to true:	", synced.length);
+
+
+				// add new rows
+				var rows = parseData.rows;
+
+
+
+				//console.log("adding missing rows: 				", rows.length);
+
+				for (var i = 0; i < rows.length; i++) {
+
+					var row = rows[i];
+
+					//console.log(row);
+
+					// --
+
+					// perform real query here
+
+					var columnArray 	= JSON.parse( row.columnValues );
+
+					var query 			= new querySQL( );
+
+					query.type			= row.operation;
+
+					for (var a = 0; a < columnArray.length; a++) {
+
+						var column = new Array();
+						
+						column.name 	= false;
+						column.value	= columnArray[a];
+
+						query.values.push( column );
+
+					}
+
+					query.sync = false;
+
+					database.executeQuery( row.queryString, query );
+
+
+					// ---
+
+
+					var query = new querySQL( );
+
+					query.type  = "insert";
+
+					query.table = "sync_client";
+
+					query.setValue( "original_id", row.original_id );
+
+					query.setValue( "queryString", row.queryString );
+
+					query.setValue( "columnValues", row.columnValues ); 
+
+					query.setValue( "operation", row.operation ); 
+
+					query.setValue( "clusterClientID", row.clusterClientID ); 
+
+					query.setValue( "server_id", row.id ); 
+
+					query.setValue( "synced", true );
+
+
+
+					query.sync = false;
+
+
+
+					var result = database.query( query );
+
+				}
+
+				
+
+			break;
+		}
+
+
+
+	}
+
+
+}
+
+
+/*
+var syncClientManager = new syncClient();
+
+
+readline.emitKeypressEvents( process.stdin );
+process.stdin.setRawMode(true);
+
+process.stdin.on('keypress', (str, key) => {
+  //console.log(str)
+  //console.log(key)
+  console.log(key);
+
+  if(key.name == "c") {
+
+  	process.exit();
+
+  }
+  	
+
+	var exampleQuery = new querySQL( );
+
+	exampleQuery.type  = "insert";
+
+	exampleQuery.table = "comment";
+
+	exampleQuery.setValue( "author_id", 1123 );
+
+
+
+	var insert = database.query( exampleQuery );
+
+	var queryString	= database.query( exampleQuery, false );
+
+	syncClientManager.sendMessage( queryString, exampleQuery, insert.lastInsertRowid );
+
+})
+
+
+*/

framework/sync/syncDownloadDatabase.js

@@ -0,0 +1,18 @@
+/*
+
+	Copyright (c) 2020, 2023, The Unified Company.
+
+	This code is part of Unify.
+
+	This program is free software; you can redistribute it and/or modify
+	it under the terms of the ESA Software Community License - Strong Copyleft,
+
+	https://unifyjs.org 
+
+*/
+
+import syncClient                     	from './syncClient.js';
+
+var SyncClient = new syncClient("download_database");
+
+

framework/sync/syncServer.js

@@ -0,0 +1,454 @@
+/*
+
+	Copyright (c) 2020, 2023, The Unified Company.
+
+	This code is part of Unify.
+
+	This program is free software; you can redistribute it and/or modify
+	it under the terms of the ESA Software Community License - Strong Copyleft,
+
+	https://unifyjs.org 
+
+*/
+
+import { WebSocketServer } 				from 'ws';
+
+import http 							from 'http';
+
+import https 							from 'https';
+
+import fs 								from 'fs';
+
+import objectManager 					from '../server/objectManager.js';
+
+import Console 							from '../server/console.js';
+
+import webConsole						from '../unify/console.js';
+
+import socketMessage 					from '../unify/socketMessage.js';
+
+import querySQL 						from '../unify/querySQL.js';
+
+import database 						from '../server/database.js';
+
+import sqlite 							from '../server/sqlite.js';
+
+import path 							from 'path';
+
+import sqlite3							from 'better-sqlite3';
+
+
+database.createTable( "sync_server" );
+
+
+database.addColumn( "original_id", "INTEGER", "sync_server" );
+
+database.addColumn( "queryString", "TEXT", "sync_server" );
+
+database.addColumn( "columnValues", "TEXT", "sync_server" );
+
+database.addColumn( "operation", "TEXT", "sync_server" );
+
+database.addColumn( "clusterClientID", "INTEGER", "sync_server" );
+
+
+
+database.createTable( "sync_client" );
+
+
+database.addColumn( "original_id", "INTEGER", "sync_client" );
+
+database.addColumn( "server_id", "INTEGER", "sync_client" );
+
+database.addColumn( "queryString", "TEXT", "sync_client" );
+
+database.addColumn( "columnValues", "TEXT", "sync_client" );
+
+database.addColumn( "operation", "TEXT", "sync_client" );
+
+database.addColumn( "clusterClientID", "INTEGER", "sync_client" );
+
+database.addColumn( "synced", "BOOLEAN", "sync_client" );
+
+
+
+
+
+class socketClient{
+
+	socket;
+
+	userID 			= 0;
+
+	async createSocket( ) {
+
+		this.createEventListeners();
+
+		var messageID 	= 0;
+
+		var message 	= new socketMessage();
+
+		message.id 		= messageID;
+		message.name 	= "connected"; 
+		message.type 	= "promise"; 
+		message.clientID 	= this.userID;
+
+		this.send( message );
+	
+	}
+
+	destroy() {
+	
+		console.log( "client disconnected ", this.userID );
+		
+		this.socketManager.removeClient( this.userID );
+	
+	}
+	
+	createEventListeners() {
+	
+		var client = this;
+
+		this.socket.on('message', async function( json ){ await client.message( json, client ) });
+		this.socket.on('close', function( json ){ client.destroy(); });
+	
+	}
+	
+	send( socketMessage ) {
+
+		this.socket.send( JSON.stringify( socketMessage ) );
+	
+	}
+
+	getRowsAfterDatetime( datetime, tablename ) {
+
+		var query = new querySQL( );
+		
+		query.type	= "select";
+		query.table	= "sync_server";
+		query.order	= ["datetime"];
+
+		query.find( "table_name", tablename );
+
+		query.find( "datetime", datetime, ">" );
+		
+		var result = database.query( query );
+
+	}
+
+	getRowsFromIDTilNow( fromID, excludeClusterClientID ) {
+
+		//console.log( "Get rows id > ", fromID, " and ClusterClientID != ", excludeClusterClientID );
+
+		var query = new querySQL( );
+		
+		query.type	= "select";
+		query.table	= "sync_server";
+		//query.order	= ["id"];
+
+		query.find( "id", fromID, ">" );
+
+		query.find( "clusterClientID", excludeClusterClientID, "!=" );
+		
+
+		return database.query( query );
+
+	}
+	
+	async message( json ) {
+
+		var object = JSON.parse( json );
+
+		switch( object.type ) {
+
+			case "add_sqlite_operation":
+
+				var unsynced = object.unsynced;
+
+				if(unsynced.length > 0) {
+
+					console.log("recieved unsynced", unsynced.length);
+
+					console.log("add row with object.clusterClientID: ", object.clusterClientID);
+
+				}
+
+
+				// todo : feature - implement insertMany - better-sqlite3 make sql query static
+				// So database doesnt have to generate query for every row.
+
+				// Better is to make an database.js cache that does this for all query's in unify
+				// that are alike so everything in unify will get an Significant speedup
+
+				for (var i = 0; i < unsynced.length; i++) {
+
+					var newRow = unsynced[i];
+
+					var query = new querySQL( );
+
+					query.type	= "insert";
+
+					query.table	= "sync_server";
+
+
+					query.setValue( "original_id", newRow.original_id );
+
+					query.setValue( "queryString", newRow.queryString );
+
+					query.setValue( "columnValues", newRow.columnValues );
+
+					query.setValue( "operation", newRow.operation );
+
+					query.setValue( "clusterClientID", newRow.clusterClientID );
+
+
+					var result = database.query( query ); 
+
+
+/*
+					// perform real query here
+
+					var columnArray 	= JSON.parse( newRow.columnValues );
+
+					var query 			= new querySQL( );
+
+					query.type			= newRow.operation;
+
+					for (var a = 0; a < columnArray.length; a++) {
+
+						var column = new Array();
+						
+						column.name 	= false;
+						column.value	= columnArray[a];
+
+						query.values.push( column );
+
+					}
+
+					query.sync = false;
+
+					database.executeQuery( newRow.queryString, query );
+
+
+					// ---
+
+					var query = new querySQL( );
+
+					query.type  = "insert";
+
+					query.table = "sync_client";
+
+					query.setValue( "original_id", newRow.original_id );
+
+					query.setValue( "queryString", newRow.queryString );
+
+					query.setValue( "columnValues", newRow.columnValues ); 
+
+					query.setValue( "operation", newRow.operation ); 
+
+					query.setValue( "clusterClientID", 555 ); // prevent loading when new db is apperent
+
+					query.setValue( "server_id", newRow.id ); 
+
+
+					query.setValue( "synced", true );
+
+					query.sync = false;
+
+
+
+					var result = database.query( query );
+*/
+
+				}
+
+
+
+				var message 			= new socketMessage();
+
+				message.name 			= "updateSyncClient";
+
+				message.clientID 		= this.userID;
+
+
+
+				message.rows 			= this.getRowsFromIDTilNow( object.lastID, object.clusterClientID );
+				//console.log("get all rows not of clusterID: ", object.clusterClientID);
+
+				//message.rows 			= [];
+
+				message.synced			= unsynced;
+
+				//console.log("rows found", message.rows.length);
+
+				this.send( message );
+
+				
+
+
+			break;
+
+			case "download_database":
+
+				console.log("download_database");
+
+				var message 			= new socketMessage();
+
+				message.name 			= "upload_database";
+
+				message.binary 			= fs.readFileSync("./framework/db/syncServer.sqlite")
+
+				this.send( message );
+
+				//console.log(databaseBinary);
+
+			break;
+
+		}
+
+	}
+	
+}
+
+
+
+
+
+class socketManager{
+	
+	socket;
+
+	core;
+	
+	userID 				= 0;
+	
+	port				= 4422;
+
+	host				= "localhost";
+
+	clients				= new Array();
+
+
+	constructor( port ){
+	          
+		if( port ) {
+		  
+			this.port = port;
+			//Console.log("start server at port:", port );
+		
+		} 
+
+		global.socketManager = this;
+		
+	}
+
+	emptyTable() {
+
+		database.database.prepare( "DELETE from sync_server" ).run()
+
+	}
+
+
+	create() {
+
+		console.log("create socket server after this");
+
+		if( !global.ssl ) {
+
+			this.socket = new WebSocketServer({ port: this.port });
+
+			console.log("socket server started on port:		" , this.port  );
+
+		} else {
+
+			this.socket 		= new WebSocketServer({ server: this.httpsServer });
+
+			console.log("started Secure socket server	", this.port);
+			
+		}
+		
+		this.createEventListeners(); 
+		       
+	}
+
+	createEventListeners() {
+	
+		var socketManager = this;
+		
+		this.socket.on('connection', function( socket ){ socketManager.connection( socket ) });
+		
+	}
+
+	addClient( id, socket ) {
+	
+		var newClient 				= new socketClient( this.port );
+		
+		newClient.userID			= id;
+		
+		newClient.socket 			= socket;
+
+		newClient.socketManager 	= this;
+
+		newClient.createSocket();
+
+		this.clients.push( newClient );
+	
+	}
+	
+	removeClient( userID ) {
+		
+		var clients = this.clients;
+
+		//this.objectManager.removeObjectsByClientID( userID );
+
+		for(var c = 0; c<clients.length;c++ ) {
+		
+			var client = clients[c];
+			
+			if( client.userID == userID ) {
+			
+				clients.splice( c, 1 );
+			
+			}
+			
+		}
+		
+	}
+
+	connection( socket ) {
+
+		const userID = this.userID++;
+
+		this.addClient( userID,  socket );
+ 
+		console.log( "client connected : ", userID );
+		
+		console.log( "number connected clients ", this.clients.length );
+
+	}
+
+}
+
+
+var socketManagerServer = new socketManager();
+
+socketManagerServer.create();
+
+
+	// just in case some user like using "kill"
+	process.on("SIGTERM", (signal) => {
+		console.log(`Process ${process.pid} received a SIGTERM signal`);
+
+		socketManagerServer.emptyTable();
+		
+		process.exit(0);
+	});
+
+	// catch ctrl-c, so that event 'exit' always works
+	process.on("SIGINT", (signal) => {
+		console.log(`Process ${process.pid} has been interrupted`);
+
+		socketManagerServer.emptyTable();
+
+		process.exit(0);
+	});