diff --git a/.gitignore b/.gitignore index c0a0e5a..1d8568d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,9 @@ lib lib64 pyvenv.cfg *.pem -*.csv \ No newline at end of file +*.csv +/build/* +Python/build/* +Python/dist/* +*.spec +dist/* \ No newline at end of file diff --git a/Moulinette/data_path_config.py b/Moulinette/data_path_config.py index 900e8a0..7da95e6 100644 --- a/Moulinette/data_path_config.py +++ b/Moulinette/data_path_config.py @@ -470,7 +470,8 @@ def main(): exists = True existing = a + 1 logging.debug("Asset " + row_asset['asset_name'] + " not created because it already exists. Processing it under the existing asset.") - excel_parser(row_asset['asset_type'], existing, row_asset["asset_name"], allowed_name_characters, dsh, dsh_global) + print(f"Excel parsing for {row_asset['asset_type']}") + excel_parser(row_asset['asset_type'], existing, row_asset["asset_name"], allowed_name_characters, dsh, dsh_global, row_device) break if exists != True: diff --git a/Python/carling_cube.py b/Python/carling_cube.py new file mode 100644 index 0000000..c03bf05 --- /dev/null +++ b/Python/carling_cube.py @@ -0,0 +1,190 @@ +import csv +import paramiko +import time +from cube_activate_ssh import activate_ssh +from dotenv import load_dotenv +import os +import re +from azure.iot.hub import IoTHubRegistryManager +from azure.iot.hub.models import Twin, TwinProperties + +load_dotenv(override=True) + +ip_address_prefix = "10.81.35." + +csv_filename = "carling.csv" +SITE_NAME = "CARLING" + +ssh_username = os.getenv("DEFAULT_CUBE_LINUX_ADMIN_USER") +ssh_password = os.getenv("DEFAULT_CUBE_LINUX_ADMIN_PASSWORD") +CONNECTION_STRING = str(os.getenv("CONNECTION_STRING_INOX_PROD")) + +def execute_ssh_command(ip, command): + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + try: + client.connect(ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False) + stdin, stdout, stderr = client.exec_command(command) + result = stdout.read().decode().lower().strip() + return result + except Exception as e: + print(f"SSH Error: {str(e)}") + raise + finally: + client.close() + +def update_cloud_config(ip, new_content): + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + try: + client.connect(ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False) + stdin, stdout, stderr = client.exec_command(f'sudo -S bash -c \'cat > /etc/cube/config-azure.properties << EOF\n{new_content}\nEOF\'\n') + stdin.write(ssh_password + "\n") + stdin.flush() + stdoutput = [line for line in stdout] + stderroutput = [line for line in stderr] + for output in stdoutput: + print(output.strip()) + except Exception as e: + print(f"SSH Error: {str(e)}") + raise + finally: + client.close() + +def restart_cloudagent(ip): + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + try: + client.connect(ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False) + stdin, stdout, stderr = client.exec_command(f'sudo -S bash -c \'systemctl restart cube-web-cloudagent << EOF\n\nEOF\'\n') + stdin.write(ssh_password + "\n") + stdin.flush() + stdoutput = [line for line in stdout] + stderroutput = [line for line in stderr] + for output in stdoutput: + print(output.strip()) + except Exception as e: + print(f"SSH Error: {str(e)}") + raise + finally: + client.close() + +def main(): + + print("Starting...") + + with open(csv_filename, mode="w", newline="") as file: + writer = csv.writer(file) + writer.writerow(["Number", "IP address", "Cube ID", "Environment", "Correct configuration"]) + + registry_manager = IoTHubRegistryManager.from_connection_string(CONNECTION_STRING) + + results = [] + + numbers = range(65, 76) + for i in numbers: + ip_address = f"{ip_address_prefix}{i}" + print(f"Activating SSH for {ip_address}:", end=" ") + + try: + activate_ssh(ip_address) + except Exception as e: + print("Failed!") + writer.writerow([i, ip_address, "UNREACHABLE (HTTPS)", "NA", "NA"]) + file.flush() + continue + + print("Activated!") + + ssh_command = "hostname" + print(f"Executing {ssh_command} for {ip_address}:", end=" ") + try: + cube_id = execute_ssh_command(ip_address, ssh_command) + except Exception as e: + print("Failed!") + writer.writerow([i, ip_address, "UNREACHABLE (SSH)", "NA", "NA"]) + file.flush() + continue + print(cube_id) + + print(f"Getting configured Connection String") + try: + connection_string = execute_ssh_command(ip_address, "grep \"connection-string\" /etc/cube/config-azure.properties") + if connection_string == "": + raise Exception("No Connection String extracted!") + + iothub_match = re.search(r"hostname\\=(.*?);", connection_string, re.IGNORECASE) + iothub = iothub_match.group(1) if iothub_match else None + if iothub.lower() == "IotHub-CUBE-PROD.azure-devices.net".lower(): + migration = "SAFT" + elif iothub.lower() == "iot-ingest-ess-prod.azure-devices.net".lower(): + migration = "INOX" + else: + migration = "NONE" + + device_id_match = re.search(r"deviceid\\=(.*?);", connection_string, re.IGNORECASE) + cloud_cube_id = device_id_match.group(1) if device_id_match else None + if cloud_cube_id.lower() == cube_id.lower(): + status = "CORRECT" + else: + status = "INCORRECT" + + except Exception as e: + print(e) + migration = "NONE" + status = "INCORRECT" + + if migration == "SAFT" or migration == "NONE": + print("SAFT device, migrating to INOX...") + + print("Creating device on INOX...") + try: + registry_manager.create_device_with_sas( + cube_id, + primary_key="", secondary_key="", + status="enabled", + iot_edge=False + ) + except Exception as iot_e: + print("Error creating new device!") + print(iot_e) + continue + + print("Adding tags to new device...") + try: + twin = registry_manager.get_twin(cube_id) + twin_patch = Twin(properties=TwinProperties(desired={}), tags={ + "site": SITE_NAME, + "number": i + }) + registry_manager.update_twin(cube_id, twin_patch, twin.etag) + except Exception as iot_e: + print("Error assigning tags to new device!") + print(iot_e) + + print("Requesting primary key...") + try: + device_info = registry_manager.get_device(cube_id) + primary_key = device_info.authentication.symmetric_key.primary_key + new_connection_string = f"HostName\\={CONNECTION_STRING.split(';')[0].split('=')[1]};DeviceId\\={cube_id};SharedAccessKey\\={primary_key}" + new_content = f'light-telemetry=false\ncompression-enabled=true\ntelemetry-on=true\nremote-update-on=true\nconnection-string={new_connection_string}' + except Exception as iot_e: + print("Error getting new Connection String!") + print(iot_e) + continue + print("Setting new Connection String...") + try: + update_cloud_config(ip_address, new_content) + except Exception as ssh_e: + print("Error when setting the new Connection String!") + print(ssh_e) + continue + print("Done!") + + restart_cloudagent(ip_address) + + writer.writerow([i, ip_address, cube_id, migration, status]) + file.flush() + +if __name__ == "__main__": + main() diff --git a/Python/create_iot_hub_devices_from_excel.py b/Python/create_iot_hub_devices_from_excel.py index 1006613..0aab780 100644 --- a/Python/create_iot_hub_devices_from_excel.py +++ b/Python/create_iot_hub_devices_from_excel.py @@ -10,8 +10,8 @@ load_dotenv() # --- Configuration --- IOTHUB_CONNECTION_STRING = os.getenv("CONNECTION_STRING_INOX_PROD") -INPUT_EXCEL_FILE = 'BELL_DEVICE_LIST.xlsx' -OUTPUT_EXCEL_FILE = 'BELL_DEVICE_LIST_CS.xlsx' +INPUT_EXCEL_FILE = 'SALINAS_DEVICE_LIST.xlsx' +OUTPUT_EXCEL_FILE = 'SALINAS_DEVICE_LIST_CS.xlsx' def get_connection_string(device: Device, host_name: str) -> str: """ diff --git a/Python/cube_activate_ssh.py b/Python/cube_activate_ssh.py index e354f9e..26f6fde 100644 --- a/Python/cube_activate_ssh.py +++ b/Python/cube_activate_ssh.py @@ -1,14 +1,23 @@ import requests import json -import argparse -import sys import os import urllib3 from dotenv import load_dotenv -import io import time +import sys -load_dotenv(override=True) +def resource_path(relative_path): + """ Get absolute path to resource, works for dev and for PyInstaller """ + try: + # PyInstaller creates a temp folder and stores path in _MEIPASS + base_path = sys._MEIPASS + except Exception: + base_path = os.path.abspath(".") + + return os.path.join(base_path, relative_path) + +dotenv_path = resource_path('.env') +load_dotenv(dotenv_path=dotenv_path) def authenticate(base_url): """ @@ -17,9 +26,17 @@ def authenticate(base_url): """ auth_url = f"{base_url}/api/auth" - username = os.getenv("DEFAULT_CUBE_WEB_ADMIN_USER") - password = os.getenv("DEFAULT_CUBE_WEB_ADMIN_PASSWORD") - certificate = os.getenv("DEFAULT_CERTIFICATE").encode("utf-8") + ENV_WEB = { + "DEFAULT_CUBE_WEB_ADMIN_USER": os.getenv("DEFAULT_CUBE_WEB_ADMIN_USER"), + "DEFAULT_CUBE_WEB_ADMIN_PASSWORD": os.getenv("DEFAULT_CUBE_WEB_ADMIN_PASSWORD"), + "DEFAULT_CERTIFICATE": os.getenv("DEFAULT_CERTIFICATE") + } + + username = ENV_WEB["DEFAULT_CUBE_WEB_ADMIN_USER"] + password = ENV_WEB["DEFAULT_CUBE_WEB_ADMIN_PASSWORD"] + certificate = ENV_WEB["DEFAULT_CERTIFICATE"].encode("utf-8") + + #print(f"{username} {password} {certificate}") # Prepare the multipart form data auth_params = { @@ -30,7 +47,7 @@ def authenticate(base_url): "params": (None, json.dumps(auth_params), "application/json"), "certificate": ("certificate.pem", certificate, "application/octet-stream") } - print(f"Authenticating against {auth_url} with {username} {password} {certificate}") + print(f"Authenticating against {auth_url}") try: response = requests.post(auth_url, files=files, verify=False, timeout=10) response.raise_for_status() # Raise exception for 4XX/5XX responses diff --git a/Python/cube_ssh_batch_restart_service.py b/Python/cube_ssh_batch_restart_service.py index fdcc55a..a7fd9c1 100644 --- a/Python/cube_ssh_batch_restart_service.py +++ b/Python/cube_ssh_batch_restart_service.py @@ -4,16 +4,33 @@ from cube_activate_ssh import activate_ssh from dotenv import load_dotenv import os import re +import sys -load_dotenv(override=True) +def resource_path(relative_path): + """ Get absolute path to resource, works for dev and for PyInstaller """ + try: + # PyInstaller creates a temp folder and stores path in _MEIPASS + base_path = sys._MEIPASS + except Exception: + base_path = os.path.abspath(".") -ip_address_prefix = "10.162.48." + return os.path.join(base_path, relative_path) + +dotenv_path = resource_path('.env') +load_dotenv(dotenv_path=dotenv_path) + +ip_address_prefix = "10.188.10." ssh_command = "hostname" -csv_filename = "lacabana01.csv" +csv_filename = "HOOHANA_01.csv" -ssh_username = os.getenv("DEFAULT_CUBE_LINUX_ADMIN_USER") -ssh_password = os.getenv("DEFAULT_CUBE_LINUX_ADMIN_PASSWORD") +ENV_SSH = { + "DEFAULT_CUBE_LINUX_ADMIN_USER": os.getenv("DEFAULT_CUBE_LINUX_ADMIN_USER"), + "DEFAULT_CUBE_LINUX_ADMIN_PASSWORD": os.getenv("DEFAULT_CUBE_LINUX_ADMIN_PASSWORD") +} + +ssh_username = ENV_SSH["DEFAULT_CUBE_LINUX_ADMIN_USER"] +ssh_password = ENV_SSH["DEFAULT_CUBE_LINUX_ADMIN_PASSWORD"] def execute_ssh_command(ip, command): client = paramiko.SSHClient() @@ -68,6 +85,7 @@ def restart_cloudagent(ip): def main(): print("Starting...") + #print(f"{ssh_username} {ssh_password}") with open(csv_filename, mode="w", newline="") as file: writer = csv.writer(file) @@ -125,55 +143,6 @@ def main(): migration = "NONE" status = "INCORRECT" - # if migration == "SAFT" or migration == "NONE": - # print("SAFT device, migrating to INOX...") - - # print("Creating device on INOX...") - # try: - # registry_manager.create_device_with_sas( - # cube_id, - # primary_key="", secondary_key="", - # status="enabled", - # iot_edge=False - # ) - # except Exception as iot_e: - # print("Error creating new device!") - # print(iot_e) - # continue - - # print("Adding tags to new device...") - # try: - # twin = registry_manager.get_twin(cube_id) - # twin_patch = Twin(properties=TwinProperties(desired={}), tags={ - # "site": SITE_NAME, - # "number": i - # }) - # registry_manager.update_twin(cube_id, twin_patch, twin.etag) - # except Exception as iot_e: - # print("Error assigning tags to new device!") - # print(iot_e) - - # print("Requesting primary key...") - # try: - # device_info = registry_manager.get_device(cube_id) - # primary_key = device_info.authentication.symmetric_key.primary_key - # new_connection_string = f"HostName\\={CONNECTION_STRING.split(';')[0].split('=')[1]};DeviceId\\={cube_id};SharedAccessKey\\={primary_key}" - # new_content = f'light-telemetry=false\ncompression-enabled=true\ntelemetry-on=true\nremote-update-on=true\nconnection-string={new_connection_string}' - # except Exception as iot_e: - # print("Error getting new Connection String!") - # print(iot_e) - # continue - # print("Setting new Connection String...") - # try: - # update_cloud_config(ip_address, new_content) - # except Exception as ssh_e: - # print("Error when setting the new Connection String!") - # print(ssh_e) - # continue - # print("Done!") - - restart_cloudagent(ip_address) - writer.writerow([i, ip_address, cube_id, migration, status]) file.flush() diff --git a/Python/requirements.txt b/Python/requirements.txt index 5c564c0..5a2a92f 100644 --- a/Python/requirements.txt +++ b/Python/requirements.txt @@ -2,4 +2,6 @@ azure-core azure-iot-hub python-dotenv paramiko -requests \ No newline at end of file +requests +pandas +openpyxl \ No newline at end of file