diff --git a/Db-Script/DatabaseScript.ipynb b/Db-Script/DatabaseScript.ipynb new file mode 100644 index 0000000..68bf25b --- /dev/null +++ b/Db-Script/DatabaseScript.ipynb @@ -0,0 +1,931 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

Forecast Project

" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

Database

" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Import packages for DB" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import yfinance as yfin\n", + "import sqlite3 as sql\n", + "import pandas as pd\n", + "import os\n", + "import datetime\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

YFianance API

\n", + "

Function using the yfinance API to get all market data

" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Tickers" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "#array with all the studied currencies (yahoo finance currency names)\n", + "currencies = [\"BTC-USD\",\"ETH-USD\",\"TTE.PA\"] " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def getMetaData(s : str) -> dict :\n", + " \"\"\"Get the meta information for a Ticker (currency)\n", + " \n", + " Args : \n", + " s (str) : name of the Ticker\n", + " \n", + " Return : \n", + " dict : return all the meta information of the Ticker, the dict is the following : \n", + " - symbol : str ~> name online (yahoo finance) of the stock (is also the ticker name used by the yfinance API)\n", + " - instrumentType : str ~> type of stock, can be a cryptocurrency , a stock ...\n", + " - regularMarketTime int : ~> date when those informatation has been gathered in epoch/unix time \n", + " - gmtoffset int : ~> number of second of offset between the stock's market timezome and the gtm timzome\n", + " - timezone str : ~> name of the stock's market timezone\n", + " other var a disponnible but not used here (see yfinance documentation)\n", + " \"\"\" \n", + " ticker = yfin.Ticker(s)\n", + " ticker.history()\n", + " meta = ticker.history_metadata\n", + " return meta\n", + " \n", + "def getHistoryData(s : str) -> dict :\n", + " \"\"\"Get all the information since the creation of the Ticker (currency)\n", + " \n", + " Args : \n", + " s (str) : name of the Ticker\n", + " \n", + " Return : \n", + " dict : return all the information of the Ticker since creation, the dict contaign the following information : {Date,Open,Hight,Low,Close,Volume}\n", + " \"\"\"\n", + " ticker = yfin.Ticker(s)\n", + " data = ticker.history(\"max\", interval = '1d')\n", + " data['Date'] = data.index\n", + " data = data[[\"Date\", \"Open\", \"High\", \"Low\", \"Close\", \"Volume\"]]\n", + " data.reset_index(drop=True, inplace=True)\n", + " return data\n", + "\n", + "def getPeriodData(s : str , period : str) -> dict :\n", + " \"\"\"Get the information of the Ticker for a certain period\n", + " \n", + " Args : \n", + " s (str) : name of the Ticker\n", + " period (str) : duration of the period , the information period get are ending to the current day and start a period back can take value like {1d , 5d , 1m , 1y , ...}\n", + " \n", + " Return : \n", + " dict : return all the information of the Ticker for the period, the dict contaign the following information : {Date,Open,Hight,Low,Close,Volume}\n", + " \"\"\"\n", + " ticker = yfin.Ticker(s)\n", + " data = ticker.history(period = period ,interval = '1d')\n", + " data['Date'] = data.index\n", + " data = data[[\"Date\", \"Open\", \"High\",\"Low\", \"Close\", \"Volume\"]]\n", + " data.reset_index(drop=True, inplace=True)\n", + " return data" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

DataBase Gestion

" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

Currencies_Metadata table

\n", + "

This table contaign all the usefull metadata of each currencies fetched. It contaigns :

\n", + "\n", + "It is possible to \n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "def createCurrencyMetadataTable():\n", + " \"\"\"Create a new currency table \"\"\"\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + "\n", + " requestCreate = '''CREATE TABLE IF NOT EXISTS CURRENCIES_METADATA (\n", + " [currencyId] INTEGER PRIMARY KEY,\n", + " [currencyName] TEXT, \n", + " [instrumentType] TEXT,\n", + " [regularMarketTime] TEXT,\n", + " [gtmOffset] INTEGER,\n", + " [timezone] TEXT\n", + " )'''\n", + " #currencyName : str ~> name online (yahoo finance) of the stock (is also the ticker name used by the yfinance API)\n", + " #instrumentType : str ~> type of stock, can be a cryptocurrency , a stock ...\n", + " #regularMarketTime int : ~> date when those informatation has been gathered in epoch/unix time \n", + " #gtmOffset int : ~> number of second of offset between the stock's market timezome and the gtm timzome\n", + " #timezone str : ~> name of the stock's market timezone\n", + " c.execute(requestCreate) # create a new currency table\n", + " db.commit()\n", + "\n", + "def getCurrencyMetadataElement(currencyId : int) -> dict :\n", + " \"\"\"Get an element of the currency table by its id\n", + "\n", + " Args:\n", + " currencyId (int): id of the currency in the table\n", + "\n", + " Returns:\n", + " dict: return the element as a dict with all metadata of a currency\n", + " \"\"\"\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " request = f'''SELECT * FROM CURRENCIES_METADATA WHERE currencyId='{currencyId}' '''\n", + " c.execute(request)\n", + " return list(c.fetchone())\n", + "\n", + "def getIdByNameCurrencyMetadata(currencyName: str) -> int :\n", + " \"\"\"Get an element id of the currency table by its name\n", + "\n", + " Args:\n", + " currencyName (str): name of the stock\n", + "\n", + " Returns:\n", + " int: return the element id\n", + " \"\"\"\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " request = f'''SELECT currencyId FROM CURRENCIES_METADATA WHERE currencyName='{currencyName}' '''\n", + " c.execute(request)\n", + " return c.fetchone()[0]\n", + "\n", + "def printCurrencyMetadataElement(currencyId : int):\n", + " \"\"\"Print an element of the currency table by its id \n", + "\n", + " Args:\n", + " currencyId (int): id of the currency in the table \n", + " \"\"\"\n", + " metaData = getCurrencyMetadataElement(currencyId)\n", + " header = f\"-------- MetaData for {metaData[1]} --------\"\n", + " nHeader = len(header)\n", + " print(header)\n", + " print(f\"\\nName : {metaData[1]}\")\n", + " print(f\"ID : {metaData[0]}\")\n", + " print(f\"instrumentType : {metaData[2]}\")\n", + " print(f\"regularMarketTime : {metaData[3]}\")\n", + " print(f\"gtmOffset : {metaData[4]}\")\n", + " print(f\"timezone : {metaData[5]}\")\n", + " footer = \"\\n\"\n", + " for i in range(nHeader): footer += \"-\"\n", + " print(footer)\n", + "\n", + "def insertCurrencyMetadataElement(currency : str):\n", + " \"\"\"Insert a new element in the currency table\n", + "\n", + " Args:\n", + " currency (str): name of the currency to fetch its metadata with the yFinance API\n", + " \"\"\"\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " metaData = getMetaData(currency)\n", + " request = f'''INSERT INTO CURRENCIES_METADATA (currencyName,instrumentType,regularMarketTime,gtmOffset,timezone)\n", + " VALUES (\n", + " '{metaData['symbol']}',\n", + " '{metaData['instrumentType']}',\n", + " '{metaData['regularMarketTime']}',\n", + " {metaData['gmtoffset']},\n", + " '{metaData['timezone']}')'''\n", + " c.execute(request)\n", + " db.commit()\n", + "\n", + "def modifyCurrencyMetadataElement(currencyId : int, newMetaData : dict):\n", + " \"\"\"Modify an element in the currency table, the element to modify must be enteriely rewrite \n", + "\n", + " Args:\n", + " currencyId (int): id of the currency to modify in the currency table\n", + " newMetaData (dict): dictionnary contaigning all the information of the currency even the ones not changed\n", + " \"\"\"\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " #All the information MUST be given in the newMetaData dict even the id and the ones not changed\n", + " request = f'''UPDATE CURRENCIES_METADATA \n", + " SET \n", + " currencyId={newMetaData['currencyId']},\n", + " currencyName='{newMetaData['symbol']}', \n", + " instrumentType='{newMetaData['instrumentType']}', \n", + " regularMarketTime='{newMetaData['regularMarketTime']}',\n", + " gtmOffset={newMetaData['gtmOffset']},\n", + " timezone='{newMetaData['timezone']}' \n", + " WHERE currencyId = {currencyId}'''\n", + " c.execute(request)\n", + " db.commit()\n", + "\n", + "def deleteCurrencyMetadataElement(currencyId : int):\n", + " \"\"\"Delete an element from the currency table\n", + "\n", + " Args:\n", + " currencyId (int): id of the currency to delete from the currency table\n", + " \"\"\"\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " request = f'''DELETE FROM CURRENCIES_METADATA\n", + " WHERE currencyId = {currencyId}'''\n", + " c.execute(request)\n", + " db.commit()\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

Others

\n", + "

Usefull an necessary function for next parts

" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def cleanNameForSQL(s : str) -> str:\n", + "\t\"\"\"Return a new string that dont contaign special char to use it as sql table name\n", + "\n", + "\tArgs:\n", + "\t\ts (str): string where to remove special char \n", + "\n", + "\tReturns:\n", + "\t\tstr: new string without special char nor spaces \n", + "\t\"\"\"\n", + "\tfor char in ['-','.','^',' ']:\n", + "\t\ts = s.replace(char,\"\")\n", + "\treturn s" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

Market Tables

\n", + "There is one market table for each currency fetched. It contaigns :\n", + "\n", + "It is possible to \n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def createCurrencyMarketTable(currencyName : str):\n", + " \"\"\"Create a new table if it dont alredy exist to contaign all the market infomation of a currency\n", + "\n", + " Args:\n", + " currencyName (str): name of the currency for which a new table is created\n", + " \"\"\"\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " tableName = cleanNameForSQL(currencyName)\n", + " #each lign in the table represent a new day and contaign :\n", + " # Date : date ~> date of the information present\n", + " # Open : float ~> value of the stock when the market open\n", + " # High : float ~> highest value reach by the stock during the market time \n", + " # Low : float ~> lowest value reach by the stock during the market time\n", + " # Close : float ~> value of the stock when the market close \n", + " # Volume : float ~> total of stock sells during the market time \n", + " createNewCurrencyTable = f'''CREATE TABLE IF NOT EXISTS {tableName}_MARKET \n", + " ([Id] INTEGER PRIMARY KEY AUTOINCREMENT,\n", + " [Date] TEXT,\n", + " [Open] NUMBER,\n", + " [High] NUMBER,\n", + " [Low] NUMBER,\n", + " [Close] NUMBER,\n", + " [Volume] INTEGER);'''\n", + " c.execute(createNewCurrencyTable)\n", + " db.commit()\n", + "\n", + "def fillCurrencyMarketTable(currency : str , period : str = None):\n", + " \"\"\"Fill a currency market table with its information\n", + "\n", + " Args:\n", + " currency (str): yahoo finance name of the currency , used to get the information with the API\n", + " period (str, optional): default value is none : all the information since create are fetched, can be defined to limit the period (for value value autorized see yfinance doc). Defaults to None.\n", + " \"\"\"\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + "\n", + " tableName = cleanNameForSQL(currency)\n", + " requestTemplate = f'''INSERT INTO {tableName}_MARKET (Date,Open,High,Low,Close,Volume)\n", + " VALUES '''\n", + "\n", + " #verify if @period has been enter as parameter\n", + " if period : data = getPeriodData(currency , period) #if yes get only period data\n", + " else : data = getHistoryData(currency) # else get all data since creation\n", + "\n", + " #for each lign in data fetched, the request add them in the table\n", + " for i in range(len(data)):\n", + " line = data.iloc[i]\n", + " #round(line[2:5],13) #all number are rounded to the 13th decimal to avoid some float type python issues \n", + " request =requestTemplate + f'''('{line['Date']}',\n", + " {line['Open']},\n", + " {line['High']},\n", + " {line['Low']},\n", + " {line['Close']},\n", + " {line['Volume']});'''\n", + "\n", + " c.execute(request)\n", + " db.commit()\n", + "\n", + "def getCurrencyMarketElementById(currency : str , elementId : int ) -> dict :\n", + " \"\"\"Get an element from a currency market table by its id \n", + "\n", + " Args:\n", + " currency (str): yahoo finance name of the currency , used to get the information with the API\n", + " elementId (int): id of the element to get \n", + "\n", + " Returns:\n", + " dict: element of the table which id is elementId \n", + " \"\"\"\n", + " tableName = cleanNameForSQL(currency)\n", + " request = f'''SELECT * FROM {tableName}_MARKET WHERE Id = {elementId} '''\n", + "\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " c.execute(request)\n", + " return c.fetchone()[0]\n", + "\n", + "def getAllCurrencyMarketElement(currency : str ) -> pd.DataFrame :\n", + " \"\"\"Get all elements from a currency market table by its id \n", + "\n", + " Args:\n", + " currency (str): yahoo finance name of the currency , used to get the information with the API\n", + "\n", + " Returns:\n", + " list: list of all elements of the table \n", + " \"\"\"\n", + " tableName = cleanNameForSQL(currency)\n", + " request = f'''SELECT Date,Open,High,Low,Close,Volume FROM {tableName}_MARKET'''\n", + "\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " c.execute(request)\n", + " res = pd.DataFrame(c.fetchall(),columns=['Date','Open','High','Low','Close','Volume'])\n", + " # for i in range(len(res)):\n", + " # res[i] = list(res[i])\n", + " return res\n", + "\n", + "def getCloseCurrencyMarketElement(currency : str ) -> list :\n", + " \"\"\"Get close elements from a currency market table by its id \n", + "\n", + " Args:\n", + " currency (str): yahoo finance name of the currency , used to get the information with the API\n", + "\n", + " Returns:\n", + " list: list of all elements of the table \n", + " \"\"\"\n", + " tableName = cleanNameForSQL(currency)\n", + " request = f'''SELECT Close FROM {tableName}_MARKET'''\n", + "\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " c.execute(request)\n", + " res = c.fetchall()\n", + " for i in range(len(res)):\n", + " res[i] = list(res[i])[0]\n", + " return res\n", + "\n", + "def getDateCurrencyMarketElement(currency : str ) -> list :\n", + " \"\"\"Get date elements from a currency market table by its id \n", + "\n", + " Args:\n", + " currency (str): yahoo finance name of the currency , used to get the information with the API\n", + "\n", + " Returns:\n", + " list: list of all elements of the table \n", + " \"\"\"\n", + " tableName = cleanNameForSQL(currency)\n", + " request = f'''SELECT Date FROM {tableName}_MARKET'''\n", + "\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " c.execute(request)\n", + " res = c.fetchall()\n", + " for i in range(len(res)):\n", + " res[i] = datetime.datetime.strptime(list(res[i])[0][2:10],\"%y-%m-%d\").date()\n", + " return res\n", + "\n", + "def insertCurrencyMarketElement(currency : str, element : dict):\n", + " \"\"\"Insert a new currency market element in the corresponding table\n", + "\n", + " Args:\n", + " currency (str): yahoo finance name of the currency\n", + " element (dict): new element to add in the table , this dict must contaign definition for {Date,Open,Hight,Low,Close,Volume}\n", + " \"\"\"\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " tableName = cleanNameForSQL(currency)\n", + " request = f'''INSERT INTO {tableName}_MARKET (Date,Open,High,Low,Close,Volume) VALUES \n", + " ('{element['Date']}',{element['Open']},{element['High']},{element['Low']},{element['Close']},{element['Volume']})'''\n", + " c .execute(request)\n", + " db.commit()\n", + "\n", + "def printCurrencyMarketElement(currency : str , currencyId : int):\n", + " \"\"\"Print an element of the currency table by its id \n", + "\n", + " Args:\n", + " currency (str): yahoo finance name of the currency \n", + " currencyId (int): id of the currency in the table \n", + " \"\"\"\n", + " data = getCurrencyMarketElementById(currency , currencyId)\n", + " header = f\"-------- Value for {currency} ID°{currencyId} --------\\n\"\n", + " nHeader = len(header)\n", + " print(header)\n", + " print(f\"Date : {data[1]}\")\n", + " print(f\"Open : {data[2]}\")\n", + " print(f\"High : {data[3]}\")\n", + " print(f\"Low : {data[4]}\")\n", + " print(f\"Close : {data[5]}\")\n", + " print(f\"Volume : {data[6]}\")\n", + " footer = \"\\n\"\n", + " for i in range(nHeader): footer += \"-\"\n", + " print(footer)\n", + "\n", + "#No Modify or Delete function debause they currently don't seem usefull " + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

Forecast Table

\n", + "There is one forcast table for each currency fetched. It contaigns :\n", + "\n", + "It is possible to :\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "def createCurrencyForecastTable(currencyName : str):\n", + " \"\"\"Create a new table if it dont alredy exist to contaign market forecast prevision of a currency\n", + "\n", + " Args:\n", + " currencyName (str): name of the currency for which a new table is created \n", + " \"\"\"\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " tableName = cleanNameForSQL(currencyName)\n", + " #each lign in the table represent a new day and contaign :\n", + " # Date : date ~> forecast date of the information present\n", + " # Open : float ~> forecast value of the stock when the market open\n", + " # High : float ~> forecast highest value reach by the stock during the market time \n", + " # Low : float ~> forecast lowest value reach by the stock during the market time\n", + " # Close : float ~> forecast value of the stock when the market close \n", + " createNewCurrencyForecastTable = f'''CREATE TABLE IF NOT EXISTS {tableName}_FORECAST \n", + " ([Id] INTEGER PRIMARY KEY AUTOINCREMENT,\n", + " [Date] TEXT,\n", + " [Open] NUMBER,\n", + " [High] NUMBER,\n", + " [Low] NUMBER,\n", + " [Close] NUMBER)'''\n", + " c.execute(createNewCurrencyForecastTable)\n", + " db.commit()\n", + "\n", + "def getCurrencyForecastElementById(currency : str , elementId : int ) -> dict :\n", + " \"\"\"Get an from a currency forecast table by its id \n", + "\n", + " Args:\n", + " currency (str): yahoo finance name of the currency , used to get the information with the API\n", + " elementId (int): id of the element to get \n", + "\n", + " Returns:\n", + " dict: element of the table which id is elementId \n", + " \"\"\"\n", + " tableName = cleanNameForSQL(currency)\n", + " request = f'''SELECT * FROM {tableName}_FORECAST WHERE Id = {elementId} '''\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " c.execute(request)\n", + " return c.fetchone()[0]\n", + "\n", + "def insertCurrencyForecastElement(currency : str, element : dict):\n", + " \"\"\"Insert a new currency forecast element in the corresponding table \n", + "\n", + " Args:\n", + " currency (str): yahoo finance name of the currency\n", + " element (dict): new element to add in the table , this dict must contaign definition for {Date,Open,Hight,Low,Close,Volume}\n", + " \"\"\"\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " tableName = cleanNameForSQL(currency)\n", + " request = f'''INSERT INTO {tableName}_FORECAST (Date,Open,High,Low,Close,Volume) VALUES \n", + " ('{element['Date']}',{element['Open']},{element['High']},{element['Low']},{element['Close']})'''\n", + " c .execute(request)\n", + " db.commit()\n", + "\n", + "def modifyCurrencyForecastElement(currency : str , elementId : int, newElement : dict):\n", + " \"\"\"Modify an element in the currency forecast table, the element to modify must be enteriely rewrite\n", + "\n", + " Args:\n", + " currency (str): id of the currency to modify in the currency table\n", + " elementId (int): id of the currency to modify in the currency table\n", + " newElement (dict): dictionnary contaigning all the information of the currency even the ones not changed\n", + " \"\"\"\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " #All the information MUST be given in the newMetaData dict even the id and the ones not changed\n", + " tableName = cleanNameForSQL(currency)\n", + " request = f'''UPDATE {tableName}_FORECAST \n", + " SET \n", + " Date={newElement['Id']},\n", + " Open='{newElement['Open']}', \n", + " High='{newElement['High']}', \n", + " Low={newElement['Low']},\n", + " Close={newElement['Close']},\n", + " WHERE Id = {elementId}'''\n", + " c.execute(request)\n", + " db.commit()\n", + "\n", + "def deleteCurrencyForecastElement(currency : str , currencyId : int):\n", + " \"\"\"Delete an element from the currency forecast table\n", + "\n", + " Args:\n", + " currency (str): yfinance currency name \n", + " currencyId (int): id of the currency to delete from the currency table\n", + " \"\"\"\n", + " tableName = cleanNameForSQL(currency)\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " request = f'''DELETE FROM {tableName}_FORECAST \n", + " WHERE currencyId = {currencyId}'''\n", + " c.execute(request)\n", + " db.commit()\n", + "\n", + "def printCurrencyForecastElement(currency : str , currencyId : int):\n", + " \"\"\"Print an element of the currency forecast table by its id \n", + "\n", + " Args:\n", + " currency (str): yahoo finance name of the currency \n", + " currencyId (int): id of the currency in the table \n", + " \"\"\"\n", + " data = getCurrencyForecastElementById(currency , currencyId)\n", + " header = f\"-------- Forecast for {currency} ID°{currencyId} --------\\n\"\n", + " nHeader = len(header)\n", + " print(header)\n", + " print(f\"Date : {data[1]}\")\n", + " print(f\"Open : {data[2]}\")\n", + " print(f\"High : {data[3]}\")\n", + " print(f\"Low : {data[4]}\")\n", + " print(f\"Close : {data[5]}\")\n", + " footer = \"\\n\"\n", + " for i in range(nHeader): footer += \"-\"\n", + " print(footer)\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

Initialization DB Function

\n", + "

Scipt to initialize the database

" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "def setDb(currencies : list):\n", + " \"\"\"Initialize the database\n", + "\n", + " Args:\n", + " currencies (list): list of currencies to add during the db initailization\n", + " \"\"\"\n", + " createCurrencyMetadataTable()\n", + " for currency in currencies:\n", + " print(currency)\n", + " insertCurrencyMetadataElement(currency)\n", + " createCurrencyMarketTable(currency)\n", + " fillCurrencyMarketTable(currency,period = \"1y\")\n", + " createCurrencyForecastTable(currency)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "BTC-USD\n", + "INSERT INTO CURRENCIES_METADATA (currencyName,instrumentType,regularMarketTime,gtmOffset,timezone)\n", + " VALUES (\n", + " 'BTC-USD',\n", + " 'CRYPTOCURRENCY',\n", + " '2023-04-25 11:45:00+00:00',\n", + " 0,\n", + " 'UTC')\n", + "ETH-USD\n", + "INSERT INTO CURRENCIES_METADATA (currencyName,instrumentType,regularMarketTime,gtmOffset,timezone)\n", + " VALUES (\n", + " 'ETH-USD',\n", + " 'CRYPTOCURRENCY',\n", + " '2023-04-25 11:45:00+00:00',\n", + " 0,\n", + " 'UTC')\n", + "TTE.PA\n", + "INSERT INTO CURRENCIES_METADATA (currencyName,instrumentType,regularMarketTime,gtmOffset,timezone)\n", + " VALUES (\n", + " 'TTE.PA',\n", + " 'EQUITY',\n", + " '2023-04-25 13:32:36+02:00',\n", + " 7200,\n", + " 'CEST')\n" + ] + } + ], + "source": [ + "#create db\n", + "try:\n", + " os.remove(\"mydatabase.db\") #delete database if it exixts \n", + "except:\n", + " pass # if it not exists just go next\n", + "setDb(currencies)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

Display stock chart

" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "ename": "OperationalError", + "evalue": "no such table: BTCUSD_MARKET", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mOperationalError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[14], line 14\u001b[0m\n\u001b[1;32m 11\u001b[0m plt\u001b[39m.\u001b[39mplot(dateData,closeData)\n\u001b[1;32m 13\u001b[0m \u001b[39m# displayStockChart(\"BTC-USD\",\"5y\")\u001b[39;00m\n\u001b[0;32m---> 14\u001b[0m displaySimplerStockChart(\u001b[39m\"\u001b[39;49m\u001b[39mBTC-USD\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n", + "Cell \u001b[0;32mIn[14], line 5\u001b[0m, in \u001b[0;36mdisplaySimplerStockChart\u001b[0;34m(currency)\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdisplaySimplerStockChart\u001b[39m(currency: \u001b[39mstr\u001b[39m):\n\u001b[1;32m 4\u001b[0m plt\u001b[39m.\u001b[39mstyle\u001b[39m.\u001b[39muse(\u001b[39m\"\u001b[39m\u001b[39mbmh\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m----> 5\u001b[0m closeData \u001b[39m=\u001b[39m getCloseCurrencyMarketElement(currency)\n\u001b[1;32m 6\u001b[0m dateData \u001b[39m=\u001b[39m getDateCurrencyMarketElement(currency)\n\u001b[1;32m 7\u001b[0m plt\u001b[39m.\u001b[39mfigure(figsize\u001b[39m=\u001b[39m(\u001b[39m16\u001b[39m,\u001b[39m4\u001b[39m))\n", + "Cell \u001b[0;32mIn[6], line 112\u001b[0m, in \u001b[0;36mgetCloseCurrencyMarketElement\u001b[0;34m(currency)\u001b[0m\n\u001b[1;32m 110\u001b[0m db \u001b[39m=\u001b[39m sql\u001b[39m.\u001b[39mconnect(\u001b[39m\"\u001b[39m\u001b[39mmydatabase.db\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 111\u001b[0m c \u001b[39m=\u001b[39m db\u001b[39m.\u001b[39mcursor()\n\u001b[0;32m--> 112\u001b[0m c\u001b[39m.\u001b[39;49mexecute(request)\n\u001b[1;32m 113\u001b[0m res \u001b[39m=\u001b[39m c\u001b[39m.\u001b[39mfetchall()\n\u001b[1;32m 114\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39mlen\u001b[39m(res)):\n", + "\u001b[0;31mOperationalError\u001b[0m: no such table: BTCUSD_MARKET" + ] + } + ], + "source": [ + "import matplotlib.pyplot as plt \n", + "\n", + "def displaySimplerStockChart(currency: str):\n", + " plt.style.use(\"bmh\")\n", + " closeData = getCloseCurrencyMarketElement(currency)\n", + " dateData = getDateCurrencyMarketElement(currency)\n", + " plt.figure(figsize=(16,4))\n", + " plt.title(f\"{currency} Closing Prices\")\n", + " plt.xlabel('Date')\n", + " plt.ylabel('Closing Price')\n", + " plt.plot(dateData,closeData)\n", + "\n", + "# displayStockChart(\"BTC-USD\",\"5y\")\n", + "displaySimplerStockChart(\"BTC-USD\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

Database Test

" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def GeneralDatabaseTest():\n", + " #Test if db exists\n", + " print(\"--- Database General Test ---\")\n", + " if os.path.exists(\"./mydatabase.db\") : \n", + " print(\"Test 1 passed : Database exist\")\n", + " else : \n", + " print(\"Test failled : No database named `mydatabase.db` created\")\n", + " return 0\n", + " return 1\n", + "\n", + "def CurrenciesMetadatGeneralTest():\n", + " \"\"\"Assert the structure of the CURRENCIES_METADATA is good\"\"\"\n", + " print(\"\\n--- CURRENCIES_METADATA General Test ---\")\n", + " #Test if CURRENCIES_METADATA structure is good\n", + " db = sql.connect(\"mydatabase.db\")\n", + " c = db.cursor()\n", + " request ='''SELECT * FROM CURRENCIES_METADATA'''\n", + " c.execute(request)\n", + " elem = c.fetchone()\n", + " #First test number of element\n", + " if len(elem) == 6 : \n", + " print(\"Test 2 passed : Number of element in table good\")\n", + " else : \n", + " print(\"Test failed : The number of element in the CURRENCIES_METADATA table isn't correct\")\n", + " return 0\n", + " #Then test element types\n", + " types = (int , str , str , int , int , str)\n", + " testResult = True\n", + " for i in range(len(elem)):\n", + " if type(elem[i]) != types[i]:\n", + " testResult = False\n", + " print(f'Test failed : the type of the {i} element doesn\\'t correspond, it\\'s a {type(elem[i])} but should be a {types[i]}')\n", + " return 0\n", + " if testResult: print(\"Test 3 passed : Element types good\")\n", + " return 1\n", + "\n", + "def CurrenciesMetadatGetTest():\n", + " \"\"\"Assert the first element of the CURRENCIES_METADATA table correspond to the `BTC-USD` one\"\"\"\n", + " print(\"\\n--- CURRENCIES_METADATA GET Method Test ---\")\n", + " btcusd = (1 , \"BTC-USD\" , \"CRYPTOCURRENCY\" , -1 , 0 , \"UTC\")\n", + " resget = getCurrencyMetadataElement(1)\n", + " t = np.column_stack((resget , btcusd))\n", + " for i , j in t:\n", + " if not i == j and j == -1:\n", + " print(f'Test failed : the element get by the method does not correspond to the BTC-USD one the values {i} should be {j}')\n", + " return 0\n", + " print(\"Test 4 passed : The Element Fetched Correspond\")\n", + " return 1\n", + "\n", + "def CurrenciesMetadatInsertTest():\n", + " \"\"\"Assert the neawly inserted element of the CURRENCIES_METADATA table is inserted and correct\"\"\"\n", + " print(\"\\n--- CURRENCIES_METADATA INSERT Method Test ---\")\n", + " nElem = len(currencies)\n", + " newElem = (nElem , \"newElem\" , \"CRYPTOCURRENCY\" , -1 , 0 , \"UTC\")\n", + " insertCurrencyMetadataElement(\"TPA\")\n", + " resget = getCurrencyMetadataElement(nElem)\n", + " t = np.column_stack((resget , newElem))\n", + " for i , j in t:\n", + " if not i == j and j == -1:\n", + " print(f'Test failed : the element inserted isn\\'t correct : the values {i} should be {j}')\n", + " return 0\n", + " print(\"Test 5 passed : The Element Is Inserted and is Correct\")\n", + " return 1\n", + "\n", + "def TestCurrencyTable():\n", + " \"\"\"Assert all the CURRENCIES_METADATA table structure and method are functionnal\"\"\"\n", + " return CurrenciesMetadatGeneralTest() and CurrenciesMetadatGetTest() and CurrenciesMetadatInsertTest()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- Database General Test ---\n", + "Test 1 passed : Database exist\n", + "\n", + "--- CURRENCIES_METADATA General Test ---\n" + ] + }, + { + "ename": "TypeError", + "evalue": "object of type 'NoneType' has no len()", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[12], line 10\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mTests have Failed\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 8\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39m0\u001b[39m\n\u001b[0;32m---> 10\u001b[0m TestDatabase()\n", + "Cell \u001b[0;32mIn[12], line 6\u001b[0m, in \u001b[0;36mTestDatabase\u001b[0;34m()\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mTests have Failed\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 5\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39m0\u001b[39m\n\u001b[0;32m----> 6\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m TestCurrencyTable(): \n\u001b[1;32m 7\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mTests have Failed\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 8\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39m0\u001b[39m\n", + "Cell \u001b[0;32mIn[11], line 67\u001b[0m, in \u001b[0;36mTestCurrencyTable\u001b[0;34m()\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mTestCurrencyTable\u001b[39m():\n\u001b[1;32m 66\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"Assert all the CURRENCIES_METADATA table structure and method are functionnal\"\"\"\u001b[39;00m\n\u001b[0;32m---> 67\u001b[0m \u001b[39mreturn\u001b[39;00m CurrenciesMetadatGeneralTest() \u001b[39mand\u001b[39;00m CurrenciesMetadatGetTest() \u001b[39mand\u001b[39;00m CurrenciesMetadatInsertTest()\n", + "Cell \u001b[0;32mIn[11], line 21\u001b[0m, in \u001b[0;36mCurrenciesMetadatGeneralTest\u001b[0;34m()\u001b[0m\n\u001b[1;32m 19\u001b[0m elem \u001b[39m=\u001b[39m c\u001b[39m.\u001b[39mfetchone()\n\u001b[1;32m 20\u001b[0m \u001b[39m#First test number of element\u001b[39;00m\n\u001b[0;32m---> 21\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39;49m(elem) \u001b[39m==\u001b[39m \u001b[39m6\u001b[39m : \n\u001b[1;32m 22\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mTest 2 passed : Number of element in table good\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 23\u001b[0m \u001b[39melse\u001b[39;00m : \n", + "\u001b[0;31mTypeError\u001b[0m: object of type 'NoneType' has no len()" + ] + } + ], + "source": [ + "def TestDatabase():\n", + " \"\"\"Assert all the database is functionnal\"\"\"\n", + " if not GeneralDatabaseTest(): \n", + " print(\"Tests have Failed\")\n", + " return 0\n", + " if not TestCurrencyTable(): \n", + " print(\"Tests have Failed\")\n", + " return 0\n", + "\n", + "TestDatabase()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.2" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Db-Script/Db-Script.py b/Db-Script/Db-Script.py new file mode 100644 index 0000000..9a1fda4 --- /dev/null +++ b/Db-Script/Db-Script.py @@ -0,0 +1,40 @@ +##Imports + +import sqlite3 as sql +import pandas as pd +import os +import datetime +import matplotlib.pyplot as plt +import numpy as np + +def createMetadataTable(): + """Create the db table which contaign all data """ + db = sql.connect("robotgowestdb.db") + c = db.cursor() + + requestCreate = '''CREATE TABLE IF NOT EXISTS METADATA_TABLE ( + [Id] INTEGER PRIMARY KEY, + [Place] TEXT, + [Start] TEXT + )''' + c.execute(requestCreate) + db.commit() + +def insertMetadataElement(place : str , start : datetime): + """Insert a new element in the metadat table + + Args: + currency (str): name of the currency to fetch its metadata with the yFinance API + """ + db = sql.connect("robotgowestdb.db") + c = db.cursor() + request = f'''INSERT INTO METADATA_TABLE (Place,Start) + VALUES ( + '{place}', + '{start}' + )''' + c.execute(request) + db.commit() + +createMetadataTable() +insertMetadataElement("ici", datetime.datetime.now()) \ No newline at end of file diff --git a/influxdb_token.txt b/influxdb_token.txt new file mode 100644 index 0000000..999500b --- /dev/null +++ b/influxdb_token.txt @@ -0,0 +1 @@ +mlxKvuIC8vZ9qQMAVxa4jD9vCLekIMFXI3tdHmx-Ho88ER1IAQfq5J1m3Rfpz_41ML6ZRSh0jZBb-aQAzoZExA== diff --git a/robotgowestdb.db b/robotgowestdb.db new file mode 100644 index 0000000..5e0cf70 Binary files /dev/null and b/robotgowestdb.db differ