From e1e48bdb44ec1726163d6c3a7a1d2198d8734df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Gauthier?= Date: Wed, 7 Jan 2026 11:17:04 +0100 Subject: [PATCH] R analysis scripts have been created, the readme has been cleaned and reworked --- .gitignore | 1 + README.md | 155 +++++++++++++++++++++-------------- exec/plot_measures.r | 24 ++++++ exec/random_forest_predict.r | 104 +++++++++++++++++++++++ exec/sun_pos.r | 8 +- 5 files changed, 229 insertions(+), 63 deletions(-) create mode 100644 exec/plot_measures.r create mode 100644 exec/random_forest_predict.r diff --git a/.gitignore b/.gitignore index d3f6148..43b451b 100755 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ MinMaxPhoto-r.ods serial *.drawio.bkp data/* +.Rhistory # Created by https://www.toptal.com/developers/gitignore/api/platformio,c++ diff --git a/README.md b/README.md index f284838..4edb48b 100644 --- a/README.md +++ b/README.md @@ -1,86 +1,119 @@ # Arduino-Photometrics -**Arduino-photometrics** is sub-part of **Robot Go West**, this project aim to collect lighting data of location for training a prediction model for retrieve the location of the sun. +**Arduino-photometrics** is sub-module of **Robot Go West** project. +The goal of this project is to collect local lighting data to train a prediction model capable of estimating the sun's position. -The embedded part is made in **C++**, the update RTC time and the extraction data measure files are made in **Python**. +The system is structured as follows: -## Arduino mounting +* **Embedded Layer**: Developed in **C++** for real-time data collection. + +* **Data Management**: Python scripts are used to update the RTC (Real-Time Clock) and extract measurement data from the device files. + +## Arduino Mounting ![Arduino mounting](images/montage.drawio.png) -## EEPROM header and data storage -The assembly collect data and write them in the EEPROM when the embedded system is deployed. -To facilitate the understanding of the storage in the EEPROM see the following image. +## EEPROM Header and Data Storage +The system collects data and writes it to the EEPROM during deployment. +To better understand the storage structure within the EEPROM, please refer to the diagram below. ![Header data EEPROM struct](images/storage_structure.drawio.png) -## Executable files -All executables files are in `exec/` folder. +## Executable Files +All executable files are located in the `exec/` folder. ## Tutorial -In first, you need to mount the arduino (image shown higher in the Readme). -(Optional) You can download and create the support with a 3D printer from [PotPhotoResistance](https://git.cohabit.fr/gowest/gowest/src/branch/photosupport/photosupport/) made by **Jhodi Avizara**. -Now, you can simply download the project from this webpage. -Open VSCode with the PlatformIO extension on the project directory. -Verify that platformio has successfully opened the project. -Plug the arduino to your PC and check. -Run project tests graphically or with `pio test`. -Upload the project on the arduino. -Check [RTC update](#rtc-update). -Make a [Sensor calibration](#sensor-calibration). -Check that **serial_com** is well defined as false (boolean flag used to signal measurement or communication phases). -Connect your arduino near of a windows and let him take action. -After some time you can bring back the measure from the arduino to your PC. -Collect data with [Gather](#gather-measures). +### 1. Hardware Setup +* **Assembly**: First, assemble the Arduino (refer to the diagram above). +* **3D Printing (Optional)**: You can download and print the support from [PotPhotoResistance](https://git.cohabit.fr/gowest/gowest/src/branch/photosupport/photosupport/), designed by **Jhodi Avizara**. + +### 2. Software Configuration +* Download or clone the project from this repository. +* Open the project directory in **VSCode** with the **PlatformIO** extension. +* Ensure PlatformIO has successfully initialized the project. + +### 3. Testing & Deployment +* Connect the Arduino to your PC. +* Run the project tests using the GUI or the command `pio test`. +* Upload the code to the Arduino. +* Perform the [RTC update](#rtc-update) and the [Sensor calibration](#sensor-calibration). +* **Important**: Ensure the `serial_com` flag is set to `false` (this boolean flag manages the switch between measurement and communication phases). + +### 4. Data Collection +* Place the Arduino near a window and let it collect data. +* After a sufficient period, reconnect the Arduino to your PC. +* Retrieve the data using the [Gather measures](#gather-measures) script. + ### Sensor calibration -This part aim to optain the measurement range with the lower and the upper threashold of your sensor. -After the electrical mounting, place **print_min_max_res()** in the main loop function from the **SensorManager** object (compile and upload in the arduino). +The goal of this step is to determine the measurement range by identifying the lower and upper thresholds of your sensors. -Make a scale range by putting place photo sensors in the dark, for exemple with a box (darkest as possible). -Then illuminate photo sensors with the direct lightin sun or a lamp (prefere the brightess as possible). -Once the previous steps are done, write arduino min max serial printed data by print_min_max_res() into min_res and max_res for each sensor. -The soft will normalise your data for compresse them in uint_8 type (smaller information to store). +1. **Setup**: After completing the electrical assembly, call the `print_min_max_res()` method from the **SensorManager** object within the main loop (then compile and upload to the Arduino). +2. **Calibration**: + * **Dark state**: Place the light sensors in a dark environment (e.g., inside a light-proof box) to record the minimum values. + * **Bright state**: Expose the sensors to direct sunlight or a bright lamp to record the maximum values. +3. **Configuration**: Once you have the values from the Serial monitor, update the `min_res` and `max_res` variables for each sensor in your code. -**TODO:** Improve this description parts +The software will automatically normalize these values to fit into a **uint8_t** type. This compression minimizes the data size, allowing for more efficient storage in the EEPROM. -### RTC update -Before build the project, uncoment the `-D DEBUG` line in platform.ini. -Set the [communcation phase](#set-communication-phase) -Once previous steps are done, check the output on the serial port. -If the date is wrong, execute the [update time](exec/time.py) file with `python3 time.py `. - -The code permit a schedule change (winter or summer padding). -You can change the value in the [main file](main.cpp) or simply disable it. - -### Gather measures -After a measure time. -Set the [communcation phase](#set-communication-phase) - -Once the upload done, execute the [download](exec/download_csv.py) file with `python3 download_csv.py` to create a csv from collected data. -The download action remove whole arduino memory. - -### Set communication phase -Connect the arduino to your PC, set **serial_com** boolean to **True**, before uploading make you sure you don't upload when a writing datameasure phase to avoid corrupted data. +### RTC Update +1. **Enable Debug Mode**: Before building the project, uncomment the `-D DEBUG` line in `platformio.ini`. +2. **Set Phase**: Ensure the device is in the [Communication Phase](#set-communication-phase). +3. **Monitor Serial Port**: Check the serial output to verify the current date and time. +4. **Sync Time**: If the date is incorrect, run the `time.py` script located in the `exec/` folder: + ```bash + python3 exec/time.py + ``` + The Python script synchronizes the Arduino's RTC with the computer's system clock via Serial communication. -# Known limitations -## Data structure -- Issue: Storage structure work with **uint_8** measure type but not with other type (First template function doesn't compile and work properly). -## Nomadic system -- Improvement: Hardward and software are adapted to capture data with plugged into a standard electrical outlet, can be adapted for work with battery and power management. +### Gather Measures +After the data collection period is complete: +1. **Switch Mode**: Set the device to the [Communication Phase](#set-communication-phase). +2. **Download Data**: Run the `download_csv.py` script to retrieve your measurements and generate a CSV file: + ```bash + python3 exec/download_csv.py + ``` + **Important**: The download process will **wipe the entire Arduino EEPROM memory** to prepare it for the next collection cycle. -## CI pipeline gitlab -- Issue: failed for now, need to find how the serial port of the vm communicate with the pipeline interface -## Calibration phase -- Improvement: Add an assisted phase to earn upper and lower threshold values to write in the code for the normalisation and storage. +### Set Communication Phase +1. Connect the Arduino to your PC. +2. Set the `serial_com` boolean flag to `true` in the code. +3. **Warning**: To avoid data corruption, ensure the system is not actively writing measurements to the EEPROM before uploading the new configuration. + + +## Known Limitations & Future Improvements + +### Data Structure +* **Current Issue**: The storage system is optimized for `uint8_t` data types. +* **Limitation**: Generic template functions for other data types are currently non-functional. Implementing a flexible template system is planned for future versions. + +### Power Management +* **Current State**: The system is designed for a continuous power supply (standard electrical outlet). +* **Improvement**: Future hardware and software iterations will focus on battery-powered operation and low-power sleep modes to enable true portability. + +### CI/CD Pipeline (GitLab) +* **Current Issue**: The automated pipeline currently fails during virtual hardware testing. +* **Challenge**: Need to configure serial port interfacing between the Virtual Machine (VM) and the pipeline runner to enable remote hardware-in-the-loop testing. + +### Calibration Process +* **Improvement**: Develop an assisted calibration tool to automatically capture and store threshold values, replacing the current manual code update process. [CI]: https://gitlab.com/Luci_/arduino-photometrics/badges/main/pipeline.svg -# Credit -This projet are entirely made by Aurélien Gauthier. +# Prediction Model +The system is designed to monitor solar irradiance from a fixed position (e.g, my system is positioned behind a window with a **109° East-of-North** orientation). -The project use two library: - - [Low-Power](https://registry.platformio.org/libraries/rocketscream/Low-Power) for the management of sleep system. - - [DS3231 lib](https://registry.platformio.org/libraries/northernwidget/DS3231) for the management of the DS3231 RTC module. \ No newline at end of file +The goal is to map the internal lighting patterns to the sun's actual coordinates, allowing the model to estimate the sun's position based solely on photometer data. + +# Credits +This project was entirely developed by **Aurélien Gauthier**. + +The 3D model for the sensor support was designed by **Jhodi Avizara**. + +### Dependencies & Libraries +This project utilizes the following libraries: + +* [Low-Power](https://registry.platformio.org/libraries/rocketscream/Low-Power): Used for power management and system sleep cycles. +* [DS3231 Lib](https://registry.platformio.org/libraries/northernwidget/DS3231): Used for interfacing with the DS3231 RTC module. \ No newline at end of file diff --git a/exec/plot_measures.r b/exec/plot_measures.r new file mode 100644 index 0000000..fd1d831 --- /dev/null +++ b/exec/plot_measures.r @@ -0,0 +1,24 @@ +# install.packages("ragg") +# install.packages("tidyverse") +# install.packages("ggplot2") + +library(ggplot2) + +setwd("~/Documents/PlatformIO/Projects/Robot_Go_West/arduino-photometrics/exec") + +# solar <- read.csv("../data/solar_pos_data/solar_data_2026-06-01_to_2026-06-15.csv", header=TRUE) +photo <- read.csv("../data/arduino_data_package_auto_20260105_151537.csv", header=TRUE) + +photo$time <- as.POSIXct(photo$Epoch) +ggplot(data = photo, aes(x = time))+ + geom_line(aes(y = Photo_sensor0, color = "Sensor 0"))+ + geom_line(aes(y = Photo_sensor1, color = "Sensor 1"))+ + geom_line(aes(y = Photo_sensor2, color = "Sensor 2"))+ + geom_line(aes(y = Photo_sensor3, color = "Sensor 3"))+ + geom_line(aes(y = Photo_sensor4, color = "Sensor 4"))+ + geom_line(aes(y = Photo_sensor5, color = "Sensor 5"))+ + theme_minimal() + +ggplot(data = photo, aes(x = time, y = Temp_sensor0))+ + geom_line()+ + theme_minimal() diff --git a/exec/random_forest_predict.r b/exec/random_forest_predict.r new file mode 100644 index 0000000..08cb28c --- /dev/null +++ b/exec/random_forest_predict.r @@ -0,0 +1,104 @@ +install.packages('randomForest') + +library(tidyverse) +library(ggplot2) +library(lubridate) +library(dplyr) +library(randomForest) + +setwd("~/Documents/PlatformIO/Projects/Robot_Go_West/arduino-photometrics/exec") + +# Load +solar <- read.csv("../data/solar_pos_data/solar_data_2026-01-05_to_2026-01-06.csv", header=TRUE) +photo <- read.csv("../data/arduino_data_package_auto_20260105_151537.csv", header=TRUE) + +# Time type changes +photo$time <- as.POSIXct(photo$Epoch) + +photo <- photo %>% + mutate( + datetime = as.POSIXct(Epoch, origin = "1970-01-01", tz = "UTC"), + + jour = as.Date(datetime), + num_jour = as.numeric(format(datetime, "%j")), + alterative_num_jour =yday(datetime), + sin_day = sin(alterative_num_jour * (2*pi/365)), + + decimal_hour = hour(datetime) + minute(datetime)/60 + second(datetime)/3600, + rad_hour = decimal_hour * (2*pi / 24), + sin_hour = sin(rad_hour), + cos_hour = cos(rad_hour) + + # TODO: sin of the day in the year + ) + +# Transform data to improve learning during the training phase +solar$sin_azimut <- sin(solar$azimut) + +# Same +max_val_sensor = 254 +photo <- photo %>% + mutate(across(starts_with("Photo_sensor"), ~ { + .x <- (.x*-1) + max_val_sensor + .x <- as.numeric(scale(.x, center = TRUE, scale = TRUE)) + })) + +# Remove NaN colomne (i had some NaN after the application of scale at a columne entirely composed of the same value) +photo <- photo %>% + select(where(~ !all(is.na(.x)))) + +# select the nearest time raw of the sun position +max_timestamp = as.integer(max(photo$Epoch)) +min_timestamp = as.integer(min(photo$Epoch)) +elapsed_time = photo$Epoch[4] - photo$Epoch[3] + +filtered_solar <- solar %>% + filter(utime > (min_timestamp - elapsed_time) & + utime < (max_timestamp + elapsed_time)) + +remove(solar) + +# merge +binded <- bind_cols(filtered_solar, photo) + +remove(filtered_solar, photo) + +# Check elapsed time +binded$gap_time <- abs(binded$utime - binded$Epoch) + + +# Random split train and test dataset +set.seed(123) + +binded <- binded %>% mutate(id = row_number()) + +random_train_data <- binded %>% sample_frac(0.80) +random_test_data <- anti_join(binded, train_data, by = "id") + +random_train_data$id <- NULL +random_test_data$id <- NULL + +summary(random_train_data$azimut) +summary(random_test_data$azimut) + + +# Chrono split train and test dataset +# Dataset already chrono sorted + +seuil <- floor(0.80 * nrow((binded))) +chrono_train_data <- binded[1:seuil, ] +chrono_test_data <- binded[(seuil + 1):nrow(binded), ] + +summary(chrono_train_data$azimut) +summary(chrono_test_data$azimut) + +# Model creation +nb_tree = 100 +yes <- data.frame( + jour = binded$utime, + = binded$Epoch +) +random_model <- randomForest(azimut ~ , data = random_train_data, ntree = nb_tree) +chrono_model <- randomForest(azimut ~ , data = chrono_train_data, ntree = nb_tree) + + diff --git a/exec/sun_pos.r b/exec/sun_pos.r index 8547d1c..8e169a2 100644 --- a/exec/sun_pos.r +++ b/exec/sun_pos.r @@ -4,22 +4,26 @@ library(suntools) library(lubridate) + lat <- 44.7912 lon <- -0.6078 tz <- "Europe/Paris" -d_deb <- "2026-06-01" -d_fin <- "2026-06-15" +d_deb <- "2026-01-05" +d_fin <- "2026-01-06" date_debut <- as.POSIXct(d_deb, tz = tz) date_fin <- as.POSIXct(d_fin, tz = tz) sequence_temps <- seq(from = date_debut, to = date_fin, by = "15 min") +unix_time <- as.numeric(sequence_temps) + coords <- matrix(c(lon, lat), nrow = 1) positions <- solarpos(coords, sequence_temps) df_soleil <- data.frame( timestamp = sequence_temps, + utime = unix_time, azimut = positions[, 1], elevation = positions[, 2] )