From c00a57d5f6d190f9b70c50baa37e1aa2c7b60105 Mon Sep 17 00:00:00 2001 From: newtbot Date: Sun, 14 Jan 2024 02:43:27 +0800 Subject: [PATCH] iot sensor finished 1)with validation on front and backend 2)fixed seed route generating value 0 for data --- IoT-sensor/Database/locationModel.js | 75 ++++++++++++++++++ IoT-sensor/Database/mySQL.js | 34 +++++++++ IoT-sensor/Database/sensorModel.js | 110 +++++++++++++++++++++++++++ IoT-sensor/functions/dbFunctions.js | 23 ++++++ IoT-sensor/index.js | 38 ++++----- IoT-sensor/modules/IoT-sensor.js | 106 +++++++++++++++----------- Web-Server/functions/APIDatabase.js | 1 + Web-Server/functions/Database.js | 16 +++- Web-Server/functions/validateData.js | 35 ++------- Web-Server/index.js | 101 ++++++++++++++---------- Web-Server/modules/express.js | 1 + Web-Server/routes/SeedsensorData.js | 25 +++--- 12 files changed, 417 insertions(+), 148 deletions(-) create mode 100644 IoT-sensor/Database/locationModel.js create mode 100644 IoT-sensor/Database/mySQL.js create mode 100644 IoT-sensor/Database/sensorModel.js create mode 100644 IoT-sensor/functions/dbFunctions.js diff --git a/IoT-sensor/Database/locationModel.js b/IoT-sensor/Database/locationModel.js new file mode 100644 index 0000000..9163156 --- /dev/null +++ b/IoT-sensor/Database/locationModel.js @@ -0,0 +1,75 @@ +"use strict"; +const { Sequelize, DataTypes } = require("sequelize"); +const { sequelize } = require("./mySQL"); +const { isAlphaNumericwithSpaces } = require('../../Web-Server/functions/validateData') + +//sequelize.sync(); +const locationModel = sequelize.define( + "location", + { + id: { + type: DataTypes.INTEGER, + allowNull: true, + primaryKey: true, + autoIncrement: true, + validate: { + isNumeric: true, + }, + }, + name: { + type: DataTypes.STRING, + allowNull: false, + length: 10, + unique: true, + validate: { + notEmpty: true, + len: [1, 20], + //accept only alphanumeric and spaces + isAlphaNumericwithSpaces(value){ + if(!isAlphaNumericwithSpaces(value)){ + throw new Error('Invalid characters in name') + } + } + }, + }, + added_by: { + type: DataTypes.STRING, + allowNull: false, + length: 10, + validate: { + notEmpty: true, + len: [1, 20], + is: ["^[a-z0-9]+$", "i"], + isIn: [['admin', 'system' , 'Admin', 'System']], + }, + }, + description: { + type: DataTypes.STRING, + allowNull: true, + length: 100, + validate: { + notEmpty: true, + len: [1, 100], + /* + //will not validate this and fail it + "hello@123" (contains a symbol) + "" (empty string) + */ + is: ["^[a-zA-Z0-9 ]+$", "i"] + }, + }, + createdAt: { + type: DataTypes.DATE, + allowNull: true, + }, + updatedAt: { + type: DataTypes.DATE, + allowNull: true, + }, + }, + { + timestamps: true, + } +); + +module.exports = { locationModel }; diff --git a/IoT-sensor/Database/mySQL.js b/IoT-sensor/Database/mySQL.js new file mode 100644 index 0000000..cc6f70f --- /dev/null +++ b/IoT-sensor/Database/mySQL.js @@ -0,0 +1,34 @@ +const dotenv = require("dotenv"); +const path = require('path') +require('dotenv').config({ path: path.resolve(__dirname, '../../.env') }) +const Sequelize = require("sequelize"); +const fs = require('fs'); + +const sequelize = new Sequelize( + "eco_saver", + process.env.DB_USER, + process.env.DB_PASS, + { + host: "mpsqldatabase.mysql.database.azure.com", + dialect: 'mysql', + // attributeBehavior?: 'escape' | 'throw' | 'unsafe-legacy'; + attributeBehavior: 'escape', + dialectOptions: { + ssl: { + ca: fs.readFileSync(path.resolve(__dirname, '../../cert/DigiCertGlobalRootCA.crt.pem')), + }, + + }, + }, + + +); + +sequelize.authenticate().then(() => { + console.log('Connection has been established successfully.'); +}).catch((error) => { + console.error('Unable to connect to the database: ', error); +}); + +module.exports = { sequelize }; + diff --git a/IoT-sensor/Database/sensorModel.js b/IoT-sensor/Database/sensorModel.js new file mode 100644 index 0000000..bed1d9b --- /dev/null +++ b/IoT-sensor/Database/sensorModel.js @@ -0,0 +1,110 @@ +"use strict"; +const { Sequelize, DataTypes } = require("sequelize"); +const { sequelize } = require("./mySQL"); +const { locationModel } = require("./locationModel"); +const { + isAlphaNumericwithSpaces, + isAlphaNumericWithSpacesAndDash, + isMacAddress, +} = require("../../Web-Server/functions/validateData"); + +//sequelize.sync(); +const sensorModel = sequelize.define( + "sensors", + { + id: { + type: DataTypes.INTEGER, + allowNull: true, + primaryKey: true, + autoIncrement: true, + unique: true, + validate: { + notEmpty: true, + isNumeric: true, + }, + }, + name: { + type: DataTypes.STRING, + allowNull: false, + length: 10, + unique: true, + validate: { + notEmpty: true, + len: [1, 30], + //accept only alphanumeric and spaces + isAlphaNumericWithSpacesAndDash(value) { + if (!isAlphaNumericWithSpacesAndDash(value)) { + throw new Error("Invalid characters in name"); + } + }, + }, + }, + added_by: { + type: DataTypes.STRING, + allowNull: false, + length: 10, + validate: { + notEmpty: { msg: "Added by cannot be empty" }, + len: [1, 20], + is: ["^[a-z0-9]+$", "i"], + isIn: [["admin", "system", "Admin", "System"]], + }, + }, + mac_address: { + type: DataTypes.STRING, + allowNull: false, + length: 12, + unique: true, + validate: { + notEmpty: true, + len: [12, 18], + isMacAddress(value) { + if (!isMacAddress(value)) { + throw new Error("Invalid Mac Address"); + } + }, + }, + }, + description: { + type: DataTypes.STRING, + allowNull: true, + length: 100, + validate: { + notEmpty: true, + len: [1, 100], + isAlphaNumericwithSpaces(value) { + if (!isAlphaNumericwithSpaces(value)) { + throw new Error("Invalid characters in name"); + } + }, + }, + }, + locationid: { + type: DataTypes.INTEGER, + allowNull: true, + length: 100, + //one to many relationship + references: { + model: locationModel, + key: "id", + }, + validate: { + notEmpty: true, + isNumeric: true, + }, + }, + createdAt: { + type: DataTypes.DATE, + allowNull: true, + }, + updatedAt: { + type: DataTypes.DATE, + allowNull: true, + }, + }, + { + timestamps: true, + } +); + +module.exports = { sensorModel }; diff --git a/IoT-sensor/functions/dbFunctions.js b/IoT-sensor/functions/dbFunctions.js new file mode 100644 index 0000000..f03c5a9 --- /dev/null +++ b/IoT-sensor/functions/dbFunctions.js @@ -0,0 +1,23 @@ +const { locationModel } = require("../Database/locationModel"); +const { sensorModel } = require("../Database/sensorModel"); + +async function getLocation() { + const location = locationModel.findAll({ + raw: true, + order: [["id", "ASC"]], + attributes: ["id"], + }); + return location; +} + +async function getSensor() { + const sensor = sensorModel.findAll({ + raw: true, + order: [["id", "ASC"]], + attributes: ["id"], + }); + return sensor; +} + + +module.exports = { getLocation, getSensor }; \ No newline at end of file diff --git a/IoT-sensor/index.js b/IoT-sensor/index.js index 04e153e..b602b62 100644 --- a/IoT-sensor/index.js +++ b/IoT-sensor/index.js @@ -1,34 +1,34 @@ -const { iot_sensor_data } = require("./modules/IoT-sensor"); +const { run } = require("./modules/IoT-sensor"); const client = require("./modules/mqtt"); -function publishData() { - let data = iot_sensor_data(); - - // MQTT logic - client.publish("iot-data", JSON.stringify(data), { qos: 1 }, (err) => { - if (err) { - console.error("Error publishing message:", err); - } else { - console.log("Message published"); - } - }); +async function publishData() { + try { + const data = await run(); + console.log(data); + client.publish("iot-data", JSON.stringify(data)); + } catch (err) { + console.error(err); + } } client.on("connect", () => { - console.log("Connected to MQTT broker"); - publishData(); + console.log("Connected to MQTT broker"); + publishData(); }); client.on("end", () => { - console.log("Disconnected from MQTT broker"); - client.reconnect = true; + console.log("Disconnected from MQTT broker"); + client.reconnect = true; }); client.on("error", (err) => { - console.error("Error:", err); - client.end(); + console.error("Error:", err); + client.end(); }); //every 15 minutes setInterval(publishData, 900000); -//setInterval(publishData, 600); \ No newline at end of file +//every 1 minute +//setInterval(publishData, 60000); + + diff --git a/IoT-sensor/modules/IoT-sensor.js b/IoT-sensor/modules/IoT-sensor.js index ff77a7c..2c6916e 100644 --- a/IoT-sensor/modules/IoT-sensor.js +++ b/IoT-sensor/modules/IoT-sensor.js @@ -1,57 +1,71 @@ -/* -1) PSI metric data -2) Humidity -3) Gases (O3,NO2,SO2) -4) temperature -5) Air pressure? -6) windspeed? -8) time when data was collected / generated -*/ +const { getLocation, getSensor } = require("../functions/dbFunctions"); -/* -1) generate random data for each sensor -2) pass to mqtt broker -*/ +//class to generate random data +var dataAray = []; +class IoTdataGenerator { + constructor() { + } + async getLocationAndSensorId() { + try { + const loc = await getLocation(); + const sen = await getSensor(); + return { loc, sen }; + } catch (err) { + console.error(err); + } + } -let region = ["central", "north-east", "north", "east", "west"]; + async generateData() { + try { + const { loc, sen } = await this.getLocationAndSensorId(); + for (let i = 0; i < sen.length; i++) { + //console.log(sen[i].id); + //console.log(loc[i].id); + //console.log("you should appear 6 times only") + dataAray.push(firstDataRow(sen[i].id, loc[i].id)); + } + } catch (err) { + console.error(err); + } + return dataAray; -function generateRandomData() { - const psiData = getRandomValue(0, 500); - const humidityData = getRandomValue(0, 100); - const o3Data = getRandomValue(0, 600); //max 600 - const no2Data = getRandomValue(0, 1000); //max 1000 - const so2Data = getRandomValue(0, 1000); //max 1000 - const coData = getRandomValue(0 , 100); - const temperatureData = getRandomValue(24, 40); - const windspeedData = getRandomValue(0, 35); - const currentTime = new Date(Date.now() + 28800000) - .toISOString() - .slice(0, 19) - .replace("T", " "); - const regionData = region[Math.floor(Math.random() * region.length)]; + } +} - var json = { - psi: psiData.toFixed(0), - humidity: humidityData.toFixed(0) + "%", - o3: o3Data.toFixed(0) + "ppm", - no2: no2Data.toFixed(0) + "ppm", - so2: so2Data.toFixed(0) + "ppm", - co: coData.toFixed(0) + "ppm", - temperature: temperatureData.toFixed(0) + "°C", - windspeed: windspeedData.toFixed(0) + "km/h", - time: currentTime, - region: regionData, +//helper function to generate random data +function firstDataRow(sensorId, locationId) { + return { + sensorid: sensorId, + locationid: locationId, + measurement: { + psi: Math.floor(Math.random() * 30) + 5, + humidity: Math.floor(Math.random() * (90 - 80 + 1) + 80), + o3: Math.floor(Math.random() * (100 - 20 + 1) + 30), + no2: Math.floor(Math.random() * 30) + 5, + so2: Math.floor(Math.random() * 30) + 5, + co: Math.floor(Math.random() * 25 - 0.5), + temperature: Math.floor(Math.random() * (30 - 23 + 1) + 25), + windspeed: Math.floor(Math.random() * (10 - 1 + 1) + 1), + }, + //time stamp are auto generated by sequelize + //createdAt: convertDateToUTC(startDate), }; - return json; } -function getRandomValue(min, max) { - return Math.random() * (max - min) + min; +/* +1) get location and sensor id from db +2) loop through each sensor id and location id and generate random data and pass to mqtt +*/ + +async function run() { + let iotData = new IoTdataGenerator(); + const result = await iotData.generateData(); + console.log(result); + return result; } + + +module.exports = { run }; -function iot_sensor_data() { - return generateRandomData(); -} -module.exports = { iot_sensor_data }; diff --git a/Web-Server/functions/APIDatabase.js b/Web-Server/functions/APIDatabase.js index 64471ca..530d369 100644 --- a/Web-Server/functions/APIDatabase.js +++ b/Web-Server/functions/APIDatabase.js @@ -231,6 +231,7 @@ buildQuery = { queryString.month ); } else { + queryString.month = getMonthFromString(queryString.month) whereClause.month = sequelize.where( sequelize.fn("MONTH", sequelize.col("createdAt")), queryString.month diff --git a/Web-Server/functions/Database.js b/Web-Server/functions/Database.js index 45a61db..79d5c7b 100644 --- a/Web-Server/functions/Database.js +++ b/Web-Server/functions/Database.js @@ -1,5 +1,6 @@ const { sequelize } = require("../../Database/mySql.js"); const { api_log_Model } = require("../../Database/model/apiLogModel.js"); +const { sensorDataModel } = require("../../Database/model/sensorDataModel.js"); async function insertLogData(log){ try{ @@ -18,10 +19,23 @@ async function insertLogData(log){ (error){ console.error(error); } +} + +async function insertDatatoDB(data) { + try { + sensorDataModel.create({ + sensorid: data.sensorid, + locationid: data.locationid, + measurement: data.measurement, + }); + } + catch (error) { + console.error(error); + } } -module.exports = { insertLogData }; +module.exports = { insertLogData , insertDatatoDB}; diff --git a/Web-Server/functions/validateData.js b/Web-Server/functions/validateData.js index a7b6de5..33ad8e1 100644 --- a/Web-Server/functions/validateData.js +++ b/Web-Server/functions/validateData.js @@ -1,31 +1,6 @@ var validator = require("validator"); -// Regular expressions for data validation -const psiPattern = /^\d+$/; -const humidityPattern = /^\d+%$/; -const concentrationPattern = /^\d+ppm$/; -const temperaturePattern = /^-?\d+°C$/; -const windspeedPattern = /^\d+km\/h$/; -const timePattern = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/; -const regionPattern = /^[a-zA-Z-]+$/; - -function validateData(data) { - return ( - psiPattern.test(data.psi) && - humidityPattern.test(data.humidity) && - concentrationPattern.test(data.o3) && - concentrationPattern.test(data.no2) && - concentrationPattern.test(data.so2) && - concentrationPattern.test(data.co) && - temperaturePattern.test(data.temperature) && - windspeedPattern.test(data.windspeed) && - timePattern.test(data.time) && - regionPattern.test(data.region) - ); -} - -const dateRegex = - /^[A-Za-z]{3}, \d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/; +const dateRegex = /^[A-Za-z]{3}, \d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/; function isValidDateString(value) { return dateRegex.test(value); @@ -55,18 +30,22 @@ function isMacAddress(value) { function isJson(value) { //check if its object if(typeof value === "object"){ - console.log("its an object") return true } +} +function isNumber(value) { + if (typeof value === "number") { + return true; + } } module.exports = { - validateData, isValidDateString, isAlphaNumericwithSpaces, isAlphaNumericWithSpacesAndDash, isMacAddress, isJson, + isNumber, }; diff --git a/Web-Server/index.js b/Web-Server/index.js index 917461a..ee7fa3b 100644 --- a/Web-Server/index.js +++ b/Web-Server/index.js @@ -1,49 +1,66 @@ const { app } = require("./modules/express.js"); -const client = require("./modules/mqtt"); -const { validateData } = require("./functions/validateData.js"); -const { insertData } = require("./functions/database.js"); -/* -1) on data received, validate data -2) websocket to another server - -*/ +const client = require("./modules/mqtt"); +const { isJson, isNumber } = require("./functions/validateData.js"); +const { insertDatatoDB } = require("./functions/database.js"); // Event handlers -client.on('connect', () => { - console.log('Connected to MQTT broker'); - client.subscribe('iot-data'); - }); - - client.on('message', (topic, message) => { - //console.log(`Received message on topic ${topic}: ${message}`); - let data = JSON.parse(message); - if (validateData(data)) { - //upload to db logic here - insertData(data); - - //websocket logic here?? - - } - else { - console.log("Data is invalid"); - throw new Error("Data is invalid"); - } - }); - - client.on('error', (err) => { - console.error('Error:', err); - client.end(); - }); - - client.on('end', () => { - console.log('Disconnected from MQTT broker'); - client.reconnect = true; - } - ); - - - +client.on("connect", () => { + console.log("Connected to MQTT broker"); + client.subscribe("iot-data"); +}); +client.on("message", (topic, message) => { + try { + let datas = JSON.parse(message); + if (isJson(datas)) { + for (let key in datas) { + let data = parseInt( + datas[key].locationid + + " " + + datas[key].sensorid + + " " + + datas[key].measurement.psi + + " " + + datas[key].measurement.humidity + + " " + + datas[key].measurement.o3 + + " " + + datas[key].measurement.no2 + + " " + + datas[key].measurement.so2 + + " " + + datas[key].measurement.co + + " " + + datas[key].measurement.temperature + + " " + + datas[key].measurement.windspeed + ); + if (isNumber(data)) { + { + //pass datas to database + insertDatatoDB(datas[key]); + } + } else { + console.log("Invalid data"); + client.end(); + } + } + } else { + console.log("Invalid data"); + client.end(); + } + } catch (err) { + console.error(err); + } +}); +client.on("error", (err) => { + console.error("Error:", err); + client.end(); +}); +client.on("end", () => { + console.log("Disconnected from MQTT broker"); + client.reconnect = true; +}); diff --git a/Web-Server/modules/express.js b/Web-Server/modules/express.js index dda3077..b1db1c2 100644 --- a/Web-Server/modules/express.js +++ b/Web-Server/modules/express.js @@ -15,6 +15,7 @@ const limiter = rateLimit({ legacyHeaders: false, // Disable the `X-RateLimit-*` headers. }) + // Apply the rate limiting middleware to all requests. app.use(limiter) diff --git a/Web-Server/routes/SeedsensorData.js b/Web-Server/routes/SeedsensorData.js index 02db887..27610e0 100644 --- a/Web-Server/routes/SeedsensorData.js +++ b/Web-Server/routes/SeedsensorData.js @@ -43,7 +43,6 @@ function firstDataRow(startDate, sensorId, locationId) { sensorid: sensorId, locationid: locationId, measurement: { - //console.log(Math.floor(Math.random() * 30) + 5) psi: Math.floor(Math.random() * 30) + 5, humidity: Math.floor(Math.random() * (90 - 80 + 1) + 80), o3: Math.floor(Math.random() * (100 - 20 + 1) + 30), @@ -77,17 +76,19 @@ function nextDataRow(currentRow, interval) { } function numberWithinPercent(inputNumber) { - //random percent with max of 1 and min of -1 - const percent = Math.random() * 1 - Math.random(); - - const range = inputNumber * percent; - - const randomOffset = Math.random() * range; - - const newNumber = inputNumber + randomOffset; - - return Math.floor(newNumber); -} + // Define a reasonable range for the random offset + const maxOffset = 5; + const minOffset = -5; + + // Add a random offset within the defined range + const randomOffset = Math.random() * (maxOffset - minOffset) + minOffset; + + // Calculate the new number with the offset + const newNumber = inputNumber + randomOffset; + + // Ensure the new number is within a reasonable range + return Math.max(5, Math.min(100, Math.floor(newNumber))); + } //add seed router.post("/new", async (req, res, next) => {