Skip to content

Commit 5efcb5a

Browse files
freQniKfreQniK
committed
Add table scheme for plans and plan_info. Create cache scripts to mine sentinel plans and plan_info data
1 parent 0159353 commit 5efcb5a

9 files changed

Lines changed: 272 additions & 4 deletions

File tree

‎meile_node_uptime.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ def connDB(self):
2222
self.db_pool = PooledDB(
2323
creator=pymysql,
2424
maxconnections=50,
25-
host=scrtsxx.RHOST,
25+
host=scrtsxx.RHOST, # Change to scrtsxx.HOST if on same server
2626
port=scrtsxx.PORT,
2727
user=scrtsxx.USERNAME,
2828
passwd=scrtsxx.PASSWORD,

‎remote_db/merge_node_ratings.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import pymysql
2+
import scrtsxx
3+
# Configuration for Server 1 (REMOTE)
4+
server1_config = {
5+
'host': scrtsxx.RHOST,
6+
'user': scrtsxx.USERNAME,
7+
'password': scrtsxx.PASSWORD,
8+
'db': scrtsxx.DB
9+
}
10+
11+
# Configuration for Server 2 (LOCAL)
12+
server2_config = {
13+
'host': scrtsxx.HOST,
14+
'user': scrtsxx.USERNAME,
15+
'password': scrtsxx.PASSWORD,
16+
'db': scrtsxx.DB
17+
}
18+
19+
20+
conn1 = pymysql.connect(**server1_config)
21+
conn2 = pymysql.connect(**server2_config)
22+
23+
try:
24+
with conn1.cursor() as cursor1:
25+
cursor1.execute("SELECT MAX(id) FROM ratings_user")
26+
max_id_server1 = cursor1.fetchone()[0] or 0
27+
28+
with conn2.cursor() as cursor2:
29+
cursor2.execute("SELECT uuid, node_address, rating, timestamp FROM ratings_user2")
30+
rows = cursor2.fetchall()
31+
32+
with conn1.cursor() as cursor1:
33+
insert_sql = """
34+
INSERT INTO ratings_user (uuid, node_address, rating, timestamp)
35+
VALUES (%s, %s, %s, %s)
36+
"""
37+
cursor1.executemany(insert_sql, rows)
38+
conn1.commit()
39+
40+
with conn2.cursor() as cursor2:
41+
cursor2.execute("DELETE FROM ratings_user")
42+
conn2.commit()
43+
44+
with conn2.cursor() as cursor2:
45+
new_autoinc = max_id_server1 + 1
46+
cursor2.execute(f"ALTER TABLE ratings_user AUTO_INCREMENT = {new_autoinc}")
47+
conn2.commit()
48+
49+
print("Data successfully transferred and auto-increment reset on Server 2.")
50+
51+
except Exception as e:
52+
print(f"An error occurred: {e}")
53+
conn1.rollback()
54+
conn2.rollback()
55+
56+
finally:
57+
conn1.close()
58+
conn2.close()

‎remote_db/scrtsxx.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
DB = "meile"
2+
USERNAME = "mathnodes"
3+
PASSWORD = "passwd"
4+
HOST = "localhost"
5+
RHOST = "remote MySQL"
6+
PORT = 3306
7+
IP_DATA_API_KEY = "key"
8+
BTCPayToken = "t0ken"
9+
BTCPAYSERVER = "https://btcpay.foo.bar"
10+
BTCPayEmail = "email@foo.bar"
11+
IP_REGISTRY_API_KEYS = ["key1",
12+
"key2",
13+
"key3"]
14+
BUSUR = "http..."
15+
BUSUR_KEY = "key"

‎schema/plan_info.sql‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
CREATE TABLE plan_info (
2+
id BIGINT PRIMARY KEY,
3+
provider_address VARCHAR(255),
4+
duration VARCHAR(64),
5+
gigabytes BIGINT,
6+
denom VARCHAR(255),
7+
amount BIGINT,
8+
status VARCHAR(50),
9+
status_at DATETIME(6)
10+
);

‎schema/plans.sql‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
CREATE TABLE Plans (
2+
id BIGINT PRIMARY KEY,
3+
provider_address VARCHAR(255),
4+
duration VARCHAR(64),
5+
gigabytes BIGINT,
6+
denom VARCHAR(255),
7+
amount BIGINT,
8+
status VARCHAR(20),
9+
status_at DATETIME(6)
10+
);

‎scrtsxx.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,9 @@
22
USERNAME = "mathnodes"
33
PASSWORD = "passwd"
44
HOST = "localhost"
5+
RHOST = "remote MySQL"
56
PORT = 3306
6-
#IP_DATA_API_KEY = "70eeefc65c73e15c06ec8e220be3f958d80a39dbd0f4f1a7b609ae5f"
77
IP_DATA_API_KEY = "key"
8-
#BTCPayToken = "zYAgx4i"
9-
#BTCPayToken = "cLe217w"
108
BTCPayToken = "t0ken"
119
BTCPAYSERVER = "https://btcpay.foo.bar"
1210
BTCPayEmail = "email@foo.bar"

‎sentinel_plan_info.py‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
import mysql.connector
2+
import requests
3+
import json
4+
from datetime import datetime
5+
6+
# Database configuration - Modify these values accordingly
7+
db_config = {
8+
'host': 'localhost',
9+
'user': 'your_username',
10+
'password': 'your_password',
11+
'database': 'your_database'
12+
}
13+
14+
# Connect to the MySQL database
15+
try:
16+
conn = mysql.connector.connect(**db_config)
17+
cursor = conn.cursor()
18+
19+
# Step 1: Fetch all 'id' values from the 'Plans' table
20+
cursor.execute("SELECT id FROM Plans")
21+
ids = cursor.fetchall()
22+
23+
# Step 2: Loop through each id and fetch data from the API
24+
for (plan_id,) in ids:
25+
url = f"https://lcd.sentinel.co/sentinel/plans/{plan_id}"
26+
print(f"Fetching data for plan ID: {plan_id}")
27+
28+
try:
29+
response = requests.get(url)
30+
response.raise_for_status() # Raise exception for HTTP errors
31+
32+
# Step 3: Parse the JSON response
33+
data = response.json()
34+
plan = data.get('plan')
35+
36+
if plan:
37+
# Step 4: Prepare data for insertion
38+
status_at_str = plan.get('status_at', '')
39+
# Truncate the fractional seconds to 6 digits and remove 'T' and 'Z'
40+
status_at_str = status_at_str.replace('T', ' ').rstrip('Z') # Replace T with space and remove 'Z'
41+
status_at_str = status_at_str[:23] # Keep only 6 digits in fractional seconds
42+
43+
# Now parse the ISO format string
44+
dt = datetime.fromisoformat(status_at_str)
45+
46+
# Convert to MySQL format
47+
status_at = dt.strftime('%Y-%m-%d %H:%M:%S.%f')
48+
price = plan['prices'][0] if plan['prices'] else {'denom': None, 'amount': None}
49+
amount = int(price['amount']) if price['amount'] else None
50+
insert_query = """
51+
INSERT INTO plan_info (
52+
id,
53+
provider_address,
54+
duration,
55+
gigabytes,
56+
denom,
57+
amount,
58+
status,
59+
status_at
60+
) VALUES (%s, %s, %s, %s, %s, %s, %s,%s)
61+
62+
"""
63+
64+
values = (
65+
plan['id'],
66+
plan['provider_address'],
67+
plan['duration'],
68+
plan['gigabytes'],
69+
price['denom'],
70+
amount,
71+
plan['status'],
72+
status_at
73+
)
74+
75+
# Step 5: Insert or update the plan data in the plan_info table
76+
cursor.execute(insert_query, values)
77+
78+
except requests.exceptions.RequestException as req_err:
79+
print(f"HTTP request failed for ID {plan_id}: {req_err}")
80+
except KeyError as key_err:
81+
print(f"Missing expected key in JSON for ID {plan_id}: {key_err}")
82+
except Exception as e:
83+
print(f"Error processing ID {plan_id}: {e}")
84+
conn.rollback()
85+
86+
# Commit all changes to the database
87+
conn.commit()
88+
89+
except mysql.connector.Error as db_err:
90+
print(f"Database connection error: {db_err}")
91+
92+
finally:
93+
# Ensure resources are released
94+
if 'cursor' in locals():
95+
cursor.close()
96+
if 'conn' in locals() and conn.is_connected():
97+
conn.close()
98+
print("Database connection closed.")

‎sentinel_plans.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import requests
2+
import mysql.connector
3+
from mysql.connector import Error
4+
from datetime import datetime
5+
6+
# 1. Connect to MySQL server
7+
try:
8+
connection = mysql.connector.connect(
9+
host='localhost',
10+
user='your_username', # Replace with your MySQL username
11+
password='your_password', # Replace with your MySQL password
12+
database='your_database_name' # Replace with your MySQL database name
13+
)
14+
if connection.is_connected():
15+
cursor = connection.cursor()
16+
17+
# 2. Fetch JSON data
18+
url = "https://lcd.sentinel.co/sentinel/plans?pagination.limit=5000"
19+
response = requests.get(url)
20+
response.raise_for_status() # Raise HTTP errors
21+
data = response.json()
22+
23+
# 3. Insert data into Plans table
24+
insert_query = """
25+
INSERT INTO Plans (
26+
id, provider_address, duration, gigabytes, denom, amount, status, status_at
27+
) VALUES (
28+
%s, %s, %s, %s, %s, %s, %s, %s
29+
)
30+
"""
31+
32+
for plan in data['plans']:
33+
status_at_str = plan.get('status_at', '')
34+
# Truncate the fractional seconds to 6 digits and remove 'T' and 'Z'
35+
status_at_str = status_at_str.replace('T', ' ').rstrip('Z') # Replace T with space and remove 'Z'
36+
status_at_str = status_at_str[:23] # Keep only 6 digits in fractional seconds
37+
38+
# Now parse the ISO format string
39+
dt = datetime.fromisoformat(status_at_str)
40+
41+
# Convert to MySQL format
42+
status_at = dt.strftime('%Y-%m-%d %H:%M:%S.%f')
43+
44+
# Extract the first price entry (since prices is an array)
45+
price = plan['prices'][0] if plan['prices'] else {'denom': None, 'amount': None}
46+
47+
# Convert gigabytes and amount to integers
48+
gigabytes = int(plan['gigabytes']) if plan['gigabytes'] else None
49+
amount = int(price['amount']) if price['amount'] else None
50+
51+
# Prepare values for insertion
52+
record = (
53+
plan['id'],
54+
plan['provider_address'],
55+
plan['duration'],
56+
gigabytes,
57+
price['denom'],
58+
amount,
59+
plan['status'],
60+
plan['status_at'] # MySQL will automatically handle fractional seconds
61+
)
62+
63+
cursor.execute(insert_query, record)
64+
65+
# Commit changes
66+
connection.commit()
67+
print(f"Inserted {cursor.rowcount} rows into the Plans table")
68+
69+
except Error as e:
70+
print(f"MySQL Error: {e}")
71+
except requests.exceptions.RequestException as e:
72+
print(f"HTTP Request Error: {e}")
73+
finally:
74+
# Close connection
75+
if 'connection' in locals() and connection.is_connected():
76+
cursor.close()
77+
connection.close()
78+
print("MySQL connection closed")

‎tests/plans.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
curl https://lcd.sentinel.co/sentinel/plans/1

0 commit comments

Comments
 (0)