Crypto

requirements.txt

@@ -9,7 +9,8 @@ python-dateutil>=2.8.0
 performer-pytorch
 tqdm>=4.36.1
 nystrom-attention
-pytorch-lightning
+pytorch-lightning==1.5
+torchtext==0.6.0
 netCDF4
 scikit-learn
 omegaconf

spacetimeformer/data/Crypto/AcquireData.sh

@@ -0,0 +1,28 @@
+
+#!/bin/sh
+echo "acquiring up-to-date data from Binance"
+wget https://www.cryptodatadownload.com/cdd/BTCUSDT_Binance_futures_data_hour.csv \
+https://www.cryptodatadownload.com/cdd/ETHUSDT_Binance_futures_data_hour.csv \
+https://www.cryptodatadownload.com/cdd/LINKUSDT_Binance_futures_data_hour.csv \
+https://www.cryptodatadownload.com/cdd/EOSUSDT_Binance_futures_data_hour.csv \
+https://www.cryptodatadownload.com/cdd/XMRUSDT_Binance_futures_data_hour.csv \
+https://www.cryptodatadownload.com/cdd/NEOUSDT_Binance_futures_data_hour.csv \
+https://www.cryptodatadownload.com/cdd/LTCUSDT_Binance_futures_data_hour.csv \
+--no-check-certificate
+sed -i '1d' LTCUSDT_Binance_futures_data_hour.csv
+sed -i '1d' BTCUSDT_Binance_futures_data_hour.csv
+sed -i '1d' ETHUSDT_Binance_futures_data_hour.csv
+sed -i '1d' LINKUSDT_Binance_futures_data_hour.csv
+sed -i '1d' EOSUSDT_Binance_futures_data_hour.csv
+sed -i '1d' XMRUSDT_Binance_futures_data_hour.csv
+sed -i '1d' NEOUSDT_Binance_futures_data_hour.csv
+echo "creating dataset"
+python ConvertData.py
+echo "cleaning up"
+rm BTCUSDT_Binance_futures_data_hour.csv
+rm ETHUSDT_Binance_futures_data_hour.csv
+rm LINKUSDT_Binance_futures_data_hour.csv
+rm EOSUSDT_Binance_futures_data_hour.csv
+rm NEOUSDT_Binance_futures_data_hour.csv
+rm LTCUSDT_Binance_futures_data_hour.csv
+rm XMRUSDT_Binance_futures_data_hour.csv

spacetimeformer/data/Crypto/ConvertData.py

@@ -0,0 +1,70 @@
+from calendar import c
+import pandas as pd
+import requests
+from datetime import datetime
+
+BTCUSDT = pd.read_csv('BTCUSDT_Binance_futures_data_hour.csv', parse_dates = ['date']).sort_values(by = 'date').reset_index(drop = True)
+ETHUSDT = pd.read_csv('ETHUSDT_Binance_futures_data_hour.csv', parse_dates = ['date']).sort_values(by = 'date').reset_index(drop = True)
+LINKUSDT = pd.read_csv('LINKUSDT_Binance_futures_data_hour.csv', parse_dates = ['date']).sort_values(by = 'date').reset_index(drop = True)
+EOSUSDT = pd.read_csv('EOSUSDT_Binance_futures_data_hour.csv', parse_dates = ['date']).sort_values(by = 'date').reset_index(drop = True)
+XMRUSDT = pd.read_csv('XMRUSDT_Binance_futures_data_hour.csv', parse_dates = ['date']).sort_values(by = 'date').reset_index(drop = True)
+NEOUSDT = pd.read_csv('NEOUSDT_Binance_futures_data_hour.csv', parse_dates = ['date']).sort_values(by = 'date').reset_index(drop = True)
+LTCUSDT = pd.read_csv('LTCUSDT_Binance_futures_data_hour.csv', parse_dates = ['date']).sort_values(by = 'date').reset_index(drop = True)
+
+
+
+ETHUSDT.rename(columns = {'open': 'ETH_open', 'high': 'ETH_high', 'low': 'ETHT_low', 'close': 'ETH_close', 'volume': 'ETH_volume', 'tradecount': 'ETH_tradecount', 'date' :'Datetime'}, inplace = True)
+BTCUSDT.rename(columns = {'open': 'BTC_open', 'high': 'BTC_high', 'low': 'BTC_low', 'close': 'BTC_close', 'volume': 'BTC_volume', 'tradecount': 'BTC_tradecount','date' :'Datetime'}, inplace = True)
+LINKUSDT.rename(columns = {'open': 'LINK_open', 'high': 'LINK_high', 'low': 'LINK_low', 'close': 'LINK_close', 'volume': 'LINK_volume', 'tradecount': 'LINK_tradecount','date' :'Datetime'}, inplace = True)
+EOSUSDT.rename(columns = {'open': 'EOS_open', 'high': 'EOS_high', 'low': 'EOS_low', 'close': 'EOS_close', 'volume': 'EOS_volume', 'tradecount': 'EOS_tradecount','date' :'Datetime'}, inplace = True)
+XMRUSDT.rename(columns = {'open': 'XMR_open', 'high': 'XMR_high', 'low': 'XMR_low', 'close': 'XMR_close', 'volume': 'XMR_volume', 'tradecount': 'XMR_tradecount','date' :'Datetime'}, inplace = True)
+NEOUSDT.rename(columns = {'open': 'NEO_open', 'high': 'NEO_high', 'low': 'NEO_low', 'close': 'NEO_close', 'volume': 'NEO_volume', 'tradecount': 'NEO_tradecount','date' :'Datetime'}, inplace = True)
+LTCUSDT.rename(columns = {'open': 'LTC_open', 'high': 'LTC_high', 'low': 'LTC_low', 'close': 'LTC_close', 'volume': 'LTC_volume', 'tradecount': 'LTC_tradecount','date' :'Datetime'}, inplace = True)
+
+# LTCUSDT.rename(columns = {'open': 'LTC_open', 'high': 'LTC_high', 'low': 'LTC_low', 'close': 'LTC_close', 'volume': 'LTC_volume', 'tradecount': 'LTC_tradecount', 'date' :'Datetime'}, inplace = True)
+df = pd.concat([ETHUSDT, BTCUSDT,LINKUSDT, EOSUSDT, XMRUSDT, NEOUSDT, LTCUSDT], axis = 1)
+#Remove all duplicate units and date column_set
+df = df.loc[:,~df.columns.duplicated()]
+# remove symbol column
+df = df.drop(columns = ['symbol',"unix"])
+
+api_url = "https://api.alternative.me/fng/?limit=0&date_format=us"
+raw_df = requests.get(api_url).json()
+
+
+timestamp = []
+value = []
+value_classification = []
+for i in raw_df["data"]:
+  timestamp.append( i["timestamp"])
+  value.append(int(i["value"]))
+  value_classification.append(i["value_classification"])
+sm = pd.DataFrame([timestamp,value]).T
+sm.columns = ["Datetime","sentiment",]
+
+
+sm["Datetime"] = sm["Datetime"].apply(lambda d: datetime.strptime(d, "%m-%d-%Y"))
+sm.index = sm.Datetime
+
+sntiments= []
+for index, row in df.iterrows():
+  date = row["Datetime"].strftime('%Y-%m-%d')
+  s = sm[sm['Datetime'].dt.strftime('%Y-%m-%d') == date]
+  sntiments.append(s["sentiment"].values[0])
+df["sntiments"] = sntiments
+count = len(df)
+df.dropna()
+
+print("cleaning data said total rows left", count, "total rows left", len(df))
+print("saving data to csv")
+df.to_csv('../crypto_dset.csv', index = False)
+
+for col in df.columns:
+  if col != "Datetime" and col != "sntiments":
+    df[col] = df[col].pct_change()
+
+df.to_csv('../crypto_dset_normalized.csv', index = False)
+
+print("done")
+print(df.columns)
+print(len(df.columns))

spacetimeformer/data/crypto_dset.csv

@@ -1,4 +1,4 @@
-Datetime,ETH_open,ETH_high,ETHT_low,ETH_close,Volume ETH,Volume USDT,ETH_tradecount,BTC_open,BTC_high,BTC_low,BTC_close,Volume BTC,BTC_tradecount,LINK_open,LINK_high,LINK_low,LINK_close,Volume LINK,LINK_tradecount,EOS_open,EOS_high,EOS_low,EOS_close,Volume EOS,EOS_tradecount,XMR_open,XMR_high,XMR_low,XMR_close,Volume XMR,XMR_tradecount,NEO_open,NEO_high,NEO_low,NEO_close,Volume NEO,NEO_tradecount,LTCUSDT_open,LTCUSDT_high,LTCUSDT_low,LTCUSDT_close,Volume LTC,LTCUSDT_tradecount,sntiments
+Datetime,ETH_open,ETH_high,ETHT_low,ETH_close,Volume ETH,Volume USDT,ETH_tradecount,BTC_open,BTC_high,BTC_low,BTC_close,Volume BTC,BTC_tradecount,LINK_open,LINK_high,LINK_low,LINK_close,Volume LINK,LINK_tradecount,EOS_open,EOS_high,EOS_low,EOS_close,Volume EOS,EOS_tradecount,XMR_open,XMR_high,XMR_low,XMR_close,Volume XMR,XMR_tradecount,NEO_open,NEO_high,NEO_low,NEO_close,Volume NEO,NEO_tradecount,LTC_open,LTC_high,LTC_low,LTC_close,Volume LTC,LTC_tradecount,sntiments
 2020-08-02 02:00:00,400.24,400.7,395.53,398.15,174181.939,69299392.80573,38932,11956.28,11980.0,11904.86,11940.59,14216.822,40786,8.26,8.404,8.233,8.305,962665.6,18096,3.348,3.41,3.342,3.4,1952385.7,10516,90.18,90.82,90.04,90.37,6823.746,2344,13.549,13.551,13.325,13.412,58250.26,3145,61.97,63.84,61.8,63.62,282626.007,20409,80
 2020-08-02 03:00:00,398.15,409.0,396.27,408.4,288534.15,116178898.52176,59625,11940.59,12037.99,11885.0,12026.41,16050.159,41254,8.305,8.42,8.286,8.351,825468.88,17557,3.4,3.5,3.375,3.482,4076025.8,18301,90.42,91.07,89.83,90.83,7859.027,2558,13.412,13.747,13.41,13.728,90946.7,6088,63.62,65.43,63.11,65.14,257149.156,19574,80
 2020-08-02 04:00:00,408.39,418.49,302.0,373.59,1363253.858,518980107.74505,226551,12026.41,12154.25,10490.0,11185.0,135655.742,323603,8.351,8.444,6.609,7.557,6446732.53,80152,3.48,3.504,2.51,2.925,14863177.3,60396,90.8,91.28,76.91,83.21,49758.326,14740,13.728,13.819,10.44,12.132,417971.26,22936,65.13,65.13,51.0,56.96,827143.158,60704,80