199 lines
7.4 KiB
Python
199 lines
7.4 KiB
Python
from paramiko import SSHClient, AutoAddPolicy
|
|
import paramiko
|
|
from cube_activate_ssh import activate_ssh
|
|
from dotenv import load_dotenv
|
|
import os
|
|
import sys
|
|
import shlex
|
|
from scp import SCPClient
|
|
import time
|
|
|
|
def resource_path(relative_path):
|
|
""" Get absolute path to resource, works for dev and for PyInstaller """
|
|
try:
|
|
# PyInstaller creates a temp folder and stores path in _MEIPASS
|
|
base_path = sys._MEIPASS
|
|
except Exception:
|
|
base_path = os.path.abspath(".")
|
|
|
|
return os.path.join(base_path, relative_path)
|
|
|
|
dotenv_path = resource_path('.env')
|
|
load_dotenv(dotenv_path=dotenv_path)
|
|
|
|
ip_address_prefix = "10.84.171." # Myrtle subnet
|
|
ip_address_range = range(3, 5) # From 1 to 57
|
|
|
|
ENV_SSH = {
|
|
"DEFAULT_CUBE_LINUX_ADMIN_USER": os.getenv("DEFAULT_CUBE_LINUX_ADMIN_USER"),
|
|
"DEFAULT_CUBE_LINUX_ADMIN_PASSWORD": os.getenv("DEFAULT_CUBE_LINUX_ADMIN_PASSWORD")
|
|
}
|
|
|
|
ssh_username = ENV_SSH["DEFAULT_CUBE_LINUX_ADMIN_USER"]
|
|
ssh_password = ENV_SSH["DEFAULT_CUBE_LINUX_ADMIN_PASSWORD"]
|
|
|
|
def execute_ssh_command(ip, command):
|
|
client = paramiko.SSHClient()
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
try:
|
|
client.connect(ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False)
|
|
stdin, stdout, stderr = client.exec_command(command)
|
|
result = stdout.read().decode().lower().strip()
|
|
return result
|
|
except Exception as e:
|
|
print(f"SSH Error: {str(e)}", flush=True)
|
|
raise
|
|
finally:
|
|
client.close()
|
|
|
|
def execute_sudo_ssh_command(ip, command):
|
|
client = paramiko.SSHClient()
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
try:
|
|
client.connect(ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False)
|
|
client.get_transport().set_keepalive(10)
|
|
|
|
quoted_command = f"bash -c {shlex.quote(command)}"
|
|
sudo_command = f"sudo -S -p '' {quoted_command}"
|
|
stdin, stdout, stderr = client.exec_command(sudo_command, timeout=60)
|
|
time.sleep(3)
|
|
stdin.write(ssh_password + '\n')
|
|
stdin.flush()
|
|
|
|
error = stderr.read().decode().strip()
|
|
if error:
|
|
raise Exception(f"sudo SSH command failed with error: {error}")
|
|
|
|
result = stdout.read().decode().lower().strip()
|
|
return result
|
|
except Exception as e:
|
|
raise
|
|
finally:
|
|
client.close()
|
|
|
|
def scp_get_file(ip, remote_path, local_path):
|
|
client = SSHClient()
|
|
client.set_missing_host_key_policy(AutoAddPolicy())
|
|
|
|
local_path = os.path.expanduser(local_path)
|
|
local_path = os.path.abspath(local_path)
|
|
|
|
local_dir = os.path.dirname(local_path)
|
|
if local_dir:
|
|
os.makedirs(local_dir, exist_ok=True)
|
|
|
|
|
|
try:
|
|
client.connect(
|
|
ip,
|
|
port=11022,
|
|
username=ssh_username,
|
|
password=ssh_password,
|
|
allow_agent=False,
|
|
look_for_keys=False
|
|
)
|
|
|
|
with SCPClient(client.get_transport()) as scp:
|
|
scp.get(remote_path, local_path)
|
|
|
|
except Exception as e:
|
|
raise
|
|
finally:
|
|
client.close()
|
|
|
|
def main():
|
|
|
|
print(f"Starting...\n", flush=True)
|
|
|
|
numbers = ip_address_range
|
|
for i in numbers:
|
|
ip_address = f"{ip_address_prefix}{i}"
|
|
print(f"[{time.ctime(time.time())}] {str(i)} ({ip_address})", end=" ", flush=True)
|
|
|
|
try:
|
|
activate_ssh(ip_address)
|
|
except Exception as e:
|
|
print(f"SSH activation failed for {ip_address}:", flush=True)
|
|
print(f"{e}", flush=True)
|
|
print(f"Skipping CUBE...", flush=True)
|
|
continue
|
|
|
|
cube_id = "NA"
|
|
|
|
try:
|
|
cube_id = execute_ssh_command(ip_address, "hostname")
|
|
print(f"{cube_id} ✅", flush=True)
|
|
except Exception as e:
|
|
print(f"cube-xxxxx ❌", flush=True)
|
|
print(f"Error getting hostname for {ip_address}:", flush=True)
|
|
print(f"{e}", flush=True)
|
|
print(f"Skipping CUBE...", flush=True)
|
|
continue
|
|
|
|
print(f"Getting small datasets:", flush=True)
|
|
metric_names = [] #["ModTmpAvg", "ModTmpMin", "ModTmpMax", "StrModTmpAvg"]
|
|
start_date = "2024-01-01"
|
|
end_date = "2026-01-01"
|
|
for metric in metric_names:
|
|
try:
|
|
print(f"\t{metric}", end=" ", flush=True)
|
|
command = "nice -n 19 influx -database cube-db -precision rfc3339 -execute \"SELECT * FROM RETENTION_CUBE_78w." + str(metric) + " WHERE time >= '" + str(start_date) + "' AND time < '" + str(end_date) + "'\" -format csv > /data/tmp/" + str(metric) + ".csv"
|
|
execute_sudo_ssh_command(ip_address, command)
|
|
print(f"✅", flush=True)
|
|
except Exception as e:
|
|
print(f"❌", flush=True)
|
|
print(f"Failed requesting {metric}:", flush=True)
|
|
print(f"{e}", flush=True)
|
|
print(f"Skipping metric...", flush=True)
|
|
continue
|
|
|
|
print(f"Downloading small datasets:", flush=True)
|
|
for metric in metric_names:
|
|
try:
|
|
print(f"\t{metric}", end=" ", flush=True)
|
|
absolute_remote_path = "/data/tmp/" + str(metric) + ".csv"
|
|
relative_local_path = "./JUILLAUME_GOURET/" + str(i).zfill(2) + str(cube_id) + "/SMALL_DATASET/" + str(metric) + ".csv"
|
|
scp_get_file(ip_address, absolute_remote_path, relative_local_path)
|
|
print(f"✅", flush=True)
|
|
except Exception as e:
|
|
print(f"❌", flush=True)
|
|
print(f"Failed downloading {metric}", flush=True)
|
|
print(f"{e}", flush=True)
|
|
print(f"Skipping metric...", flush=True)
|
|
continue
|
|
|
|
print(f"Getting medium datasets:", flush=True)
|
|
metric_names = ["SoC", "StrSoC"]
|
|
start_date = "2024-01-01"
|
|
end_date = "2026-01-01"
|
|
for metric in metric_names:
|
|
try:
|
|
print(f"\t{metric}", end=" ", flush=True)
|
|
command = "nice -n 19 influx -database cube-db -precision rfc3339 -execute \"SELECT * FROM RETENTION_CUBE_78w." + str(metric) + " WHERE time >= '" + str(start_date) + "' AND time < '" + str(end_date) + "'\" -format csv > /data/tmp/" + str(metric) + ".csv"
|
|
execute_sudo_ssh_command(ip_address, command)
|
|
print(f"✅", flush=True)
|
|
except Exception as e:
|
|
print(f"❌", flush=True)
|
|
print(f"Failed requesting {metric}:", flush=True)
|
|
print(f"{e}", flush=True)
|
|
print(f"Skipping metric...", flush=True)
|
|
continue
|
|
|
|
print(f"Downloading medium datasets:", flush=True)
|
|
for metric in metric_names:
|
|
try:
|
|
print(f"\t{metric}", end=" ", flush=True)
|
|
absolute_remote_path = "/data/tmp/" + str(metric) + ".csv"
|
|
relative_local_path = "./JUILLAUME_GOURET/" + str(i).zfill(2) + str(cube_id) + "/MEDIUM_DATASET/" + str(metric) + ".csv"
|
|
scp_get_file(ip_address, absolute_remote_path, relative_local_path)
|
|
print(f"✅", flush=True)
|
|
except Exception as e:
|
|
print(f"❌", flush=True)
|
|
print(f"Failed downloading {metric}", flush=True)
|
|
print(f"{e}", flush=True)
|
|
print(f"Skipping metric...", flush=True)
|
|
continue
|
|
|
|
if __name__ == "__main__":
|
|
main()
|